Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # vim: set syntax=python ts=4 : |
| 3 | # |
| 4 | # Copyright (c) 2018 Intel Corporation |
| 5 | # SPDX-License-Identifier: Apache-2.0 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 6 | import os |
| 7 | import contextlib |
| 8 | import string |
| 9 | import mmap |
| 10 | import sys |
| 11 | import re |
| 12 | import subprocess |
| 13 | import select |
| 14 | import shutil |
| 15 | import shlex |
| 16 | import signal |
| 17 | import threading |
| 18 | import concurrent.futures |
| 19 | from collections import OrderedDict |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 20 | import queue |
| 21 | import time |
| 22 | import csv |
| 23 | import glob |
| 24 | import concurrent |
| 25 | import xml.etree.ElementTree as ET |
| 26 | import logging |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 27 | from pathlib import Path |
| 28 | from distutils.spawn import find_executable |
| 29 | from colorama import Fore |
Martí Bolívar | 9c92baa | 2020-07-08 14:43:07 -0700 | [diff] [blame] | 30 | import pickle |
Martí Bolívar | 07dce82 | 2020-04-13 16:50:51 -0700 | [diff] [blame] | 31 | import platform |
Anas Nashif | ae61b7e | 2020-07-06 11:30:55 -0400 | [diff] [blame] | 32 | import yaml |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 33 | import json |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 34 | from multiprocessing import Lock, Process, Value |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 35 | from typing import List |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 36 | |
Anas Nashif | ae61b7e | 2020-07-06 11:30:55 -0400 | [diff] [blame] | 37 | try: |
| 38 | # Use the C LibYAML parser if available, rather than the Python parser. |
| 39 | # It's much faster. |
Anas Nashif | ae61b7e | 2020-07-06 11:30:55 -0400 | [diff] [blame] | 40 | from yaml import CSafeLoader as SafeLoader |
| 41 | from yaml import CDumper as Dumper |
| 42 | except ImportError: |
Martí Bolívar | d8698cb | 2020-07-08 14:55:14 -0700 | [diff] [blame] | 43 | from yaml import SafeLoader, Dumper |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 44 | |
| 45 | try: |
| 46 | import serial |
| 47 | except ImportError: |
| 48 | print("Install pyserial python module with pip to use --device-testing option.") |
| 49 | |
| 50 | try: |
| 51 | from tabulate import tabulate |
| 52 | except ImportError: |
| 53 | print("Install tabulate python module with pip to use --device-testing option.") |
| 54 | |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 55 | try: |
| 56 | import psutil |
| 57 | except ImportError: |
Anas Nashif | 77946fa | 2020-05-21 18:19:01 -0400 | [diff] [blame] | 58 | print("Install psutil python module with pip to run in Qemu.") |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 59 | |
Piotr Golyzniak | 8b77348 | 2021-11-02 16:13:26 +0100 | [diff] [blame] | 60 | try: |
| 61 | import pty |
| 62 | except ImportError as capture_error: |
| 63 | if os.name == "nt": # "nt" means that program is running on Windows OS |
| 64 | pass # "--device-serial-pty" option is not supported on Windows OS |
| 65 | else: |
| 66 | raise capture_error |
| 67 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 68 | ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") |
| 69 | if not ZEPHYR_BASE: |
| 70 | sys.exit("$ZEPHYR_BASE environment variable undefined") |
| 71 | |
Martí Bolívar | 9c92baa | 2020-07-08 14:43:07 -0700 | [diff] [blame] | 72 | # This is needed to load edt.pickle files. |
Martí Bolívar | 5332847 | 2021-03-26 16:18:58 -0700 | [diff] [blame] | 73 | sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts", |
| 74 | "python-devicetree", "src")) |
| 75 | from devicetree import edtlib # pylint: disable=unused-import |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 76 | |
| 77 | # Use this for internal comparisons; that's what canonicalization is |
| 78 | # for. Don't use it when invoking other components of the build system |
| 79 | # to avoid confusing and hard to trace inconsistencies in error messages |
| 80 | # and logs, generated Makefiles, etc. compared to when users invoke these |
| 81 | # components directly. |
| 82 | # Note "normalization" is different from canonicalization, see os.path. |
| 83 | canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE) |
| 84 | |
| 85 | sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/")) |
| 86 | |
Anas Nashif | e508bab | 2020-12-07 11:27:32 -0500 | [diff] [blame] | 87 | import scl |
| 88 | import expr_parser |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 89 | |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 90 | logger = logging.getLogger('twister') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 91 | logger.setLevel(logging.DEBUG) |
| 92 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 93 | |
| 94 | class ExecutionCounter(object): |
| 95 | def __init__(self, total=0): |
| 96 | self._done = Value('i', 0) |
| 97 | self._passed = Value('i', 0) |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 98 | self._skipped_configs = Value('i', 0) |
| 99 | self._skipped_runtime = Value('i', 0) |
| 100 | self._skipped_cases = Value('i', 0) |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 101 | self._error = Value('i', 0) |
| 102 | self._failed = Value('i', 0) |
| 103 | self._total = Value('i', total) |
| 104 | self._cases = Value('i', 0) |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 105 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 106 | |
| 107 | self.lock = Lock() |
| 108 | |
| 109 | @property |
| 110 | def cases(self): |
| 111 | with self._cases.get_lock(): |
| 112 | return self._cases.value |
| 113 | |
| 114 | @cases.setter |
| 115 | def cases(self, value): |
| 116 | with self._cases.get_lock(): |
| 117 | self._cases.value = value |
| 118 | |
| 119 | @property |
| 120 | def skipped_cases(self): |
| 121 | with self._skipped_cases.get_lock(): |
| 122 | return self._skipped_cases.value |
| 123 | |
| 124 | @skipped_cases.setter |
| 125 | def skipped_cases(self, value): |
| 126 | with self._skipped_cases.get_lock(): |
| 127 | self._skipped_cases.value = value |
| 128 | |
| 129 | @property |
| 130 | def error(self): |
| 131 | with self._error.get_lock(): |
| 132 | return self._error.value |
| 133 | |
| 134 | @error.setter |
| 135 | def error(self, value): |
| 136 | with self._error.get_lock(): |
| 137 | self._error.value = value |
| 138 | |
| 139 | @property |
| 140 | def done(self): |
| 141 | with self._done.get_lock(): |
| 142 | return self._done.value |
| 143 | |
| 144 | @done.setter |
| 145 | def done(self, value): |
| 146 | with self._done.get_lock(): |
| 147 | self._done.value = value |
| 148 | |
| 149 | @property |
| 150 | def passed(self): |
| 151 | with self._passed.get_lock(): |
| 152 | return self._passed.value |
| 153 | |
| 154 | @passed.setter |
| 155 | def passed(self, value): |
| 156 | with self._passed.get_lock(): |
| 157 | self._passed.value = value |
| 158 | |
| 159 | @property |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 160 | def skipped_configs(self): |
| 161 | with self._skipped_configs.get_lock(): |
| 162 | return self._skipped_configs.value |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 163 | |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 164 | @skipped_configs.setter |
| 165 | def skipped_configs(self, value): |
| 166 | with self._skipped_configs.get_lock(): |
| 167 | self._skipped_configs.value = value |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 168 | |
| 169 | @property |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 170 | def skipped_runtime(self): |
| 171 | with self._skipped_runtime.get_lock(): |
| 172 | return self._skipped_runtime.value |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 173 | |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 174 | @skipped_runtime.setter |
| 175 | def skipped_runtime(self, value): |
| 176 | with self._skipped_runtime.get_lock(): |
| 177 | self._skipped_runtime.value = value |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 178 | |
| 179 | @property |
| 180 | def failed(self): |
| 181 | with self._failed.get_lock(): |
| 182 | return self._failed.value |
| 183 | |
| 184 | @failed.setter |
| 185 | def failed(self, value): |
| 186 | with self._failed.get_lock(): |
| 187 | self._failed.value = value |
| 188 | |
| 189 | @property |
| 190 | def total(self): |
| 191 | with self._total.get_lock(): |
| 192 | return self._total.value |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 193 | |
| 194 | class CMakeCacheEntry: |
| 195 | '''Represents a CMake cache entry. |
| 196 | |
| 197 | This class understands the type system in a CMakeCache.txt, and |
| 198 | converts the following cache types to Python types: |
| 199 | |
| 200 | Cache Type Python type |
| 201 | ---------- ------------------------------------------- |
| 202 | FILEPATH str |
| 203 | PATH str |
| 204 | STRING str OR list of str (if ';' is in the value) |
| 205 | BOOL bool |
| 206 | INTERNAL str OR list of str (if ';' is in the value) |
| 207 | ---------- ------------------------------------------- |
| 208 | ''' |
| 209 | |
| 210 | # Regular expression for a cache entry. |
| 211 | # |
| 212 | # CMake variable names can include escape characters, allowing a |
| 213 | # wider set of names than is easy to match with a regular |
| 214 | # expression. To be permissive here, use a non-greedy match up to |
| 215 | # the first colon (':'). This breaks if the variable name has a |
| 216 | # colon inside, but it's good enough. |
| 217 | CACHE_ENTRY = re.compile( |
| 218 | r'''(?P<name>.*?) # name |
| 219 | :(?P<type>FILEPATH|PATH|STRING|BOOL|INTERNAL) # type |
| 220 | =(?P<value>.*) # value |
| 221 | ''', re.X) |
| 222 | |
| 223 | @classmethod |
| 224 | def _to_bool(cls, val): |
| 225 | # Convert a CMake BOOL string into a Python bool. |
| 226 | # |
| 227 | # "True if the constant is 1, ON, YES, TRUE, Y, or a |
| 228 | # non-zero number. False if the constant is 0, OFF, NO, |
| 229 | # FALSE, N, IGNORE, NOTFOUND, the empty string, or ends in |
| 230 | # the suffix -NOTFOUND. Named boolean constants are |
| 231 | # case-insensitive. If the argument is not one of these |
| 232 | # constants, it is treated as a variable." |
| 233 | # |
| 234 | # https://cmake.org/cmake/help/v3.0/command/if.html |
| 235 | val = val.upper() |
| 236 | if val in ('ON', 'YES', 'TRUE', 'Y'): |
| 237 | return 1 |
| 238 | elif val in ('OFF', 'NO', 'FALSE', 'N', 'IGNORE', 'NOTFOUND', ''): |
| 239 | return 0 |
| 240 | elif val.endswith('-NOTFOUND'): |
| 241 | return 0 |
| 242 | else: |
| 243 | try: |
| 244 | v = int(val) |
| 245 | return v != 0 |
| 246 | except ValueError as exc: |
| 247 | raise ValueError('invalid bool {}'.format(val)) from exc |
| 248 | |
| 249 | @classmethod |
| 250 | def from_line(cls, line, line_no): |
| 251 | # Comments can only occur at the beginning of a line. |
| 252 | # (The value of an entry could contain a comment character). |
| 253 | if line.startswith('//') or line.startswith('#'): |
| 254 | return None |
| 255 | |
| 256 | # Whitespace-only lines do not contain cache entries. |
| 257 | if not line.strip(): |
| 258 | return None |
| 259 | |
| 260 | m = cls.CACHE_ENTRY.match(line) |
| 261 | if not m: |
| 262 | return None |
| 263 | |
| 264 | name, type_, value = (m.group(g) for g in ('name', 'type', 'value')) |
| 265 | if type_ == 'BOOL': |
| 266 | try: |
| 267 | value = cls._to_bool(value) |
| 268 | except ValueError as exc: |
| 269 | args = exc.args + ('on line {}: {}'.format(line_no, line),) |
| 270 | raise ValueError(args) from exc |
| 271 | elif type_ in ['STRING', 'INTERNAL']: |
| 272 | # If the value is a CMake list (i.e. is a string which |
| 273 | # contains a ';'), convert to a Python list. |
| 274 | if ';' in value: |
| 275 | value = value.split(';') |
| 276 | |
| 277 | return CMakeCacheEntry(name, value) |
| 278 | |
| 279 | def __init__(self, name, value): |
| 280 | self.name = name |
| 281 | self.value = value |
| 282 | |
| 283 | def __str__(self): |
| 284 | fmt = 'CMakeCacheEntry(name={}, value={})' |
| 285 | return fmt.format(self.name, self.value) |
| 286 | |
| 287 | |
| 288 | class CMakeCache: |
| 289 | '''Parses and represents a CMake cache file.''' |
| 290 | |
| 291 | @staticmethod |
| 292 | def from_file(cache_file): |
| 293 | return CMakeCache(cache_file) |
| 294 | |
| 295 | def __init__(self, cache_file): |
| 296 | self.cache_file = cache_file |
| 297 | self.load(cache_file) |
| 298 | |
| 299 | def load(self, cache_file): |
| 300 | entries = [] |
| 301 | with open(cache_file, 'r') as cache: |
| 302 | for line_no, line in enumerate(cache): |
| 303 | entry = CMakeCacheEntry.from_line(line, line_no) |
| 304 | if entry: |
| 305 | entries.append(entry) |
| 306 | self._entries = OrderedDict((e.name, e) for e in entries) |
| 307 | |
| 308 | def get(self, name, default=None): |
| 309 | entry = self._entries.get(name) |
| 310 | if entry is not None: |
| 311 | return entry.value |
| 312 | else: |
| 313 | return default |
| 314 | |
| 315 | def get_list(self, name, default=None): |
| 316 | if default is None: |
| 317 | default = [] |
| 318 | entry = self._entries.get(name) |
| 319 | if entry is not None: |
| 320 | value = entry.value |
| 321 | if isinstance(value, list): |
| 322 | return value |
| 323 | elif isinstance(value, str): |
| 324 | return [value] if value else [] |
| 325 | else: |
| 326 | msg = 'invalid value {} type {}' |
| 327 | raise RuntimeError(msg.format(value, type(value))) |
| 328 | else: |
| 329 | return default |
| 330 | |
| 331 | def __contains__(self, name): |
| 332 | return name in self._entries |
| 333 | |
| 334 | def __getitem__(self, name): |
| 335 | return self._entries[name].value |
| 336 | |
| 337 | def __setitem__(self, name, entry): |
| 338 | if not isinstance(entry, CMakeCacheEntry): |
| 339 | msg = 'improper type {} for value {}, expecting CMakeCacheEntry' |
| 340 | raise TypeError(msg.format(type(entry), entry)) |
| 341 | self._entries[name] = entry |
| 342 | |
| 343 | def __delitem__(self, name): |
| 344 | del self._entries[name] |
| 345 | |
| 346 | def __iter__(self): |
| 347 | return iter(self._entries.values()) |
| 348 | |
| 349 | |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 350 | class TwisterException(Exception): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 351 | pass |
| 352 | |
| 353 | |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 354 | class TwisterRuntimeError(TwisterException): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 355 | pass |
| 356 | |
| 357 | |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 358 | class ConfigurationError(TwisterException): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 359 | def __init__(self, cfile, message): |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 360 | TwisterException.__init__(self, cfile + ": " + message) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 361 | |
| 362 | |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 363 | class BuildError(TwisterException): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 364 | pass |
| 365 | |
| 366 | |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 367 | class ExecutionError(TwisterException): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 368 | pass |
| 369 | |
| 370 | |
| 371 | class HarnessImporter: |
| 372 | |
| 373 | def __init__(self, name): |
Anas Nashif | e508bab | 2020-12-07 11:27:32 -0500 | [diff] [blame] | 374 | sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 375 | module = __import__("harness") |
| 376 | if name: |
| 377 | my_class = getattr(module, name) |
| 378 | else: |
| 379 | my_class = getattr(module, "Test") |
| 380 | |
| 381 | self.instance = my_class() |
| 382 | |
| 383 | |
| 384 | class Handler: |
| 385 | def __init__(self, instance, type_str="build"): |
| 386 | """Constructor |
| 387 | |
| 388 | """ |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 389 | self.state = "waiting" |
| 390 | self.run = False |
| 391 | self.duration = 0 |
| 392 | self.type_str = type_str |
| 393 | |
| 394 | self.binary = None |
| 395 | self.pid_fn = None |
| 396 | self.call_make_run = False |
| 397 | |
| 398 | self.name = instance.name |
| 399 | self.instance = instance |
| 400 | self.timeout = instance.testcase.timeout |
| 401 | self.sourcedir = instance.testcase.source_dir |
| 402 | self.build_dir = instance.build_dir |
| 403 | self.log = os.path.join(self.build_dir, "handler.log") |
| 404 | self.returncode = 0 |
| 405 | self.set_state("running", self.duration) |
| 406 | self.generator = None |
| 407 | self.generator_cmd = None |
| 408 | |
| 409 | self.args = [] |
Jingru Wang | fed1c54 | 2021-06-11 11:54:55 +0800 | [diff] [blame] | 410 | self.terminated = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 411 | |
| 412 | def set_state(self, state, duration): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 413 | self.state = state |
| 414 | self.duration = duration |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 415 | |
| 416 | def get_state(self): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 417 | ret = (self.state, self.duration) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 418 | return ret |
| 419 | |
| 420 | def record(self, harness): |
| 421 | if harness.recording: |
| 422 | filename = os.path.join(self.build_dir, "recording.csv") |
| 423 | with open(filename, "at") as csvfile: |
| 424 | cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep) |
| 425 | cw.writerow(harness.fieldnames) |
| 426 | for instance in harness.recording: |
| 427 | cw.writerow(instance) |
| 428 | |
Jingru Wang | fed1c54 | 2021-06-11 11:54:55 +0800 | [diff] [blame] | 429 | def terminate(self, proc): |
| 430 | # encapsulate terminate functionality so we do it consistently where ever |
| 431 | # we might want to terminate the proc. We need try_kill_process_by_pid |
| 432 | # because of both how newer ninja (1.6.0 or greater) and .NET / renode |
| 433 | # work. Newer ninja's don't seem to pass SIGTERM down to the children |
| 434 | # so we need to use try_kill_process_by_pid. |
| 435 | for child in psutil.Process(proc.pid).children(recursive=True): |
| 436 | try: |
| 437 | os.kill(child.pid, signal.SIGTERM) |
| 438 | except ProcessLookupError: |
| 439 | pass |
| 440 | proc.terminate() |
| 441 | # sleep for a while before attempting to kill |
| 442 | time.sleep(0.5) |
| 443 | proc.kill() |
| 444 | self.terminated = True |
| 445 | |
Piotr Golyzniak | 0e1ecb3 | 2021-09-23 00:14:04 +0200 | [diff] [blame] | 446 | def add_missing_testscases(self, harness): |
| 447 | """ |
| 448 | If testsuite was broken by some error (e.g. timeout) it is necessary to |
| 449 | add information about next testcases, which were not be |
| 450 | performed due to this error. |
| 451 | """ |
| 452 | for c in self.instance.testcase.cases: |
| 453 | if c not in harness.tests: |
| 454 | harness.tests[c] = "BLOCK" |
| 455 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 456 | |
| 457 | class BinaryHandler(Handler): |
| 458 | def __init__(self, instance, type_str): |
| 459 | """Constructor |
| 460 | |
| 461 | @param instance Test Instance |
| 462 | """ |
| 463 | super().__init__(instance, type_str) |
| 464 | |
Eugeniy Paltsev | 16032b67 | 2020-10-08 22:42:02 +0300 | [diff] [blame] | 465 | self.call_west_flash = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 466 | |
| 467 | # Tool options |
| 468 | self.valgrind = False |
| 469 | self.lsan = False |
| 470 | self.asan = False |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 471 | self.ubsan = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 472 | self.coverage = False |
| 473 | |
| 474 | def try_kill_process_by_pid(self): |
| 475 | if self.pid_fn: |
| 476 | pid = int(open(self.pid_fn).read()) |
| 477 | os.unlink(self.pid_fn) |
| 478 | self.pid_fn = None # clear so we don't try to kill the binary twice |
| 479 | try: |
| 480 | os.kill(pid, signal.SIGTERM) |
| 481 | except ProcessLookupError: |
| 482 | pass |
| 483 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 484 | def _output_reader(self, proc): |
| 485 | self.line = proc.stdout.readline() |
| 486 | |
| 487 | def _output_handler(self, proc, harness): |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 488 | if harness.is_pytest: |
| 489 | harness.handle(None) |
| 490 | return |
| 491 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 492 | log_out_fp = open(self.log, "wt") |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 493 | timeout_extended = False |
| 494 | timeout_time = time.time() + self.timeout |
| 495 | while True: |
| 496 | this_timeout = timeout_time - time.time() |
| 497 | if this_timeout < 0: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 498 | break |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 499 | reader_t = threading.Thread(target=self._output_reader, args=(proc,), daemon=True) |
| 500 | reader_t.start() |
| 501 | reader_t.join(this_timeout) |
| 502 | if not reader_t.is_alive(): |
| 503 | line = self.line |
| 504 | logger.debug("OUTPUT: {0}".format(line.decode('utf-8').rstrip())) |
| 505 | log_out_fp.write(line.decode('utf-8')) |
| 506 | log_out_fp.flush() |
| 507 | harness.handle(line.decode('utf-8').rstrip()) |
| 508 | if harness.state: |
| 509 | if not timeout_extended or harness.capture_coverage: |
| 510 | timeout_extended = True |
| 511 | if harness.capture_coverage: |
| 512 | timeout_time = time.time() + 30 |
| 513 | else: |
| 514 | timeout_time = time.time() + 2 |
| 515 | else: |
| 516 | reader_t.join(0) |
| 517 | break |
| 518 | try: |
| 519 | # POSIX arch based ztests end on their own, |
| 520 | # so let's give it up to 100ms to do so |
| 521 | proc.wait(0.1) |
| 522 | except subprocess.TimeoutExpired: |
| 523 | self.terminate(proc) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 524 | |
| 525 | log_out_fp.close() |
| 526 | |
| 527 | def handle(self): |
| 528 | |
| 529 | harness_name = self.instance.testcase.harness.capitalize() |
| 530 | harness_import = HarnessImporter(harness_name) |
| 531 | harness = harness_import.instance |
| 532 | harness.configure(self.instance) |
| 533 | |
| 534 | if self.call_make_run: |
| 535 | command = [self.generator_cmd, "run"] |
Eugeniy Paltsev | 16032b67 | 2020-10-08 22:42:02 +0300 | [diff] [blame] | 536 | elif self.call_west_flash: |
| 537 | command = ["west", "flash", "--skip-rebuild", "-d", self.build_dir] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 538 | else: |
| 539 | command = [self.binary] |
| 540 | |
| 541 | run_valgrind = False |
| 542 | if self.valgrind and shutil.which("valgrind"): |
| 543 | command = ["valgrind", "--error-exitcode=2", |
| 544 | "--leak-check=full", |
| 545 | "--suppressions=" + ZEPHYR_BASE + "/scripts/valgrind.supp", |
| 546 | "--log-file=" + self.build_dir + "/valgrind.log" |
| 547 | ] + command |
| 548 | run_valgrind = True |
| 549 | |
| 550 | logger.debug("Spawning process: " + |
| 551 | " ".join(shlex.quote(word) for word in command) + os.linesep + |
| 552 | "in directory: " + self.build_dir) |
| 553 | |
| 554 | start_time = time.time() |
| 555 | |
| 556 | env = os.environ.copy() |
| 557 | if self.asan: |
| 558 | env["ASAN_OPTIONS"] = "log_path=stdout:" + \ |
| 559 | env.get("ASAN_OPTIONS", "") |
| 560 | if not self.lsan: |
| 561 | env["ASAN_OPTIONS"] += "detect_leaks=0" |
| 562 | |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 563 | if self.ubsan: |
| 564 | env["UBSAN_OPTIONS"] = "log_path=stdout:halt_on_error=1:" + \ |
| 565 | env.get("UBSAN_OPTIONS", "") |
| 566 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 567 | with subprocess.Popen(command, stdout=subprocess.PIPE, |
| 568 | stderr=subprocess.PIPE, cwd=self.build_dir, env=env) as proc: |
| 569 | logger.debug("Spawning BinaryHandler Thread for %s" % self.name) |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 570 | t = threading.Thread(target=self._output_handler, args=(proc, harness,), daemon=True) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 571 | t.start() |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 572 | t.join() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 573 | if t.is_alive(): |
| 574 | self.terminate(proc) |
| 575 | t.join() |
| 576 | proc.wait() |
| 577 | self.returncode = proc.returncode |
Eugeniy Paltsev | 79b3a50 | 2020-12-22 20:00:01 +0300 | [diff] [blame] | 578 | self.try_kill_process_by_pid() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 579 | |
| 580 | handler_time = time.time() - start_time |
| 581 | |
| 582 | if self.coverage: |
| 583 | subprocess.call(["GCOV_PREFIX=" + self.build_dir, |
| 584 | "gcov", self.sourcedir, "-b", "-s", self.build_dir], shell=True) |
| 585 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 586 | # FIXME: This is needed when killing the simulator, the console is |
| 587 | # garbled and needs to be reset. Did not find a better way to do that. |
Eugeniy Paltsev | afb34e3 | 2020-11-29 21:28:28 +0300 | [diff] [blame] | 588 | if sys.stdout.isatty(): |
| 589 | subprocess.call(["stty", "sane"]) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 590 | |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 591 | if harness.is_pytest: |
| 592 | harness.pytest_run(self.log) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 593 | self.instance.results = harness.tests |
| 594 | |
| 595 | if not self.terminated and self.returncode != 0: |
| 596 | # When a process is killed, the default handler returns 128 + SIGTERM |
| 597 | # so in that case the return code itself is not meaningful |
| 598 | self.set_state("failed", handler_time) |
| 599 | self.instance.reason = "Failed" |
| 600 | elif run_valgrind and self.returncode == 2: |
| 601 | self.set_state("failed", handler_time) |
| 602 | self.instance.reason = "Valgrind error" |
| 603 | elif harness.state: |
| 604 | self.set_state(harness.state, handler_time) |
Anas Nashif | b802af8 | 2020-04-26 21:57:38 -0400 | [diff] [blame] | 605 | if harness.state == "failed": |
| 606 | self.instance.reason = "Failed" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 607 | else: |
| 608 | self.set_state("timeout", handler_time) |
| 609 | self.instance.reason = "Timeout" |
Piotr Golyzniak | 0e1ecb3 | 2021-09-23 00:14:04 +0200 | [diff] [blame] | 610 | self.add_missing_testscases(harness) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 611 | |
| 612 | self.record(harness) |
| 613 | |
| 614 | |
| 615 | class DeviceHandler(Handler): |
| 616 | |
| 617 | def __init__(self, instance, type_str): |
| 618 | """Constructor |
| 619 | |
| 620 | @param instance Test Instance |
| 621 | """ |
| 622 | super().__init__(instance, type_str) |
| 623 | |
| 624 | self.suite = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 625 | |
| 626 | def monitor_serial(self, ser, halt_fileno, harness): |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 627 | if harness.is_pytest: |
| 628 | harness.handle(None) |
| 629 | return |
| 630 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 631 | log_out_fp = open(self.log, "wt") |
| 632 | |
| 633 | ser_fileno = ser.fileno() |
| 634 | readlist = [halt_fileno, ser_fileno] |
| 635 | |
Andrei Emeltchenko | 10fa48d | 2021-01-12 09:56:32 +0200 | [diff] [blame] | 636 | if self.coverage: |
| 637 | # Set capture_coverage to True to indicate that right after |
| 638 | # test results we should get coverage data, otherwise we exit |
| 639 | # from the test. |
| 640 | harness.capture_coverage = True |
| 641 | |
| 642 | ser.flush() |
| 643 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 644 | while ser.isOpen(): |
| 645 | readable, _, _ = select.select(readlist, [], [], self.timeout) |
| 646 | |
| 647 | if halt_fileno in readable: |
| 648 | logger.debug('halted') |
| 649 | ser.close() |
| 650 | break |
| 651 | if ser_fileno not in readable: |
| 652 | continue # Timeout. |
| 653 | |
| 654 | serial_line = None |
| 655 | try: |
| 656 | serial_line = ser.readline() |
| 657 | except TypeError: |
| 658 | pass |
| 659 | except serial.SerialException: |
| 660 | ser.close() |
| 661 | break |
| 662 | |
| 663 | # Just because ser_fileno has data doesn't mean an entire line |
| 664 | # is available yet. |
| 665 | if serial_line: |
| 666 | sl = serial_line.decode('utf-8', 'ignore').lstrip() |
| 667 | logger.debug("DEVICE: {0}".format(sl.rstrip())) |
| 668 | |
| 669 | log_out_fp.write(sl) |
| 670 | log_out_fp.flush() |
| 671 | harness.handle(sl.rstrip()) |
| 672 | |
| 673 | if harness.state: |
Andrei Emeltchenko | 10fa48d | 2021-01-12 09:56:32 +0200 | [diff] [blame] | 674 | if not harness.capture_coverage: |
| 675 | ser.close() |
| 676 | break |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 677 | |
| 678 | log_out_fp.close() |
| 679 | |
Anas Nashif | 3b86f13 | 2020-05-21 10:35:33 -0400 | [diff] [blame] | 680 | def device_is_available(self, instance): |
| 681 | device = instance.platform.name |
| 682 | fixture = instance.testcase.harness_config.get("fixture") |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 683 | for d in self.suite.duts: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 684 | if fixture and fixture not in d.fixtures: |
Anas Nashif | 3b86f13 | 2020-05-21 10:35:33 -0400 | [diff] [blame] | 685 | continue |
Andy Ross | 098fce3 | 2021-03-02 05:13:07 -0800 | [diff] [blame] | 686 | if d.platform != device or not (d.serial or d.serial_pty): |
| 687 | continue |
| 688 | d.lock.acquire() |
| 689 | avail = False |
| 690 | if d.available: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 691 | d.available = 0 |
| 692 | d.counter += 1 |
Andy Ross | 098fce3 | 2021-03-02 05:13:07 -0800 | [diff] [blame] | 693 | avail = True |
| 694 | d.lock.release() |
| 695 | if avail: |
Anas Nashif | 2a74053 | 2021-02-04 08:08:20 -0500 | [diff] [blame] | 696 | return d |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 697 | |
Anas Nashif | 2a74053 | 2021-02-04 08:08:20 -0500 | [diff] [blame] | 698 | return None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 699 | |
| 700 | def make_device_available(self, serial): |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 701 | for d in self.suite.duts: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 702 | if d.serial == serial or d.serial_pty: |
| 703 | d.available = 1 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 704 | |
| 705 | @staticmethod |
| 706 | def run_custom_script(script, timeout): |
| 707 | with subprocess.Popen(script, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as proc: |
| 708 | try: |
| 709 | stdout, _ = proc.communicate(timeout=timeout) |
| 710 | logger.debug(stdout.decode()) |
| 711 | |
| 712 | except subprocess.TimeoutExpired: |
| 713 | proc.kill() |
| 714 | proc.communicate() |
| 715 | logger.error("{} timed out".format(script)) |
| 716 | |
| 717 | def handle(self): |
| 718 | out_state = "failed" |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 719 | runner = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 720 | |
Anas Nashif | 2a74053 | 2021-02-04 08:08:20 -0500 | [diff] [blame] | 721 | hardware = self.device_is_available(self.instance) |
| 722 | while not hardware: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 723 | logger.debug("Waiting for device {} to become available".format(self.instance.platform.name)) |
| 724 | time.sleep(1) |
Anas Nashif | 2a74053 | 2021-02-04 08:08:20 -0500 | [diff] [blame] | 725 | hardware = self.device_is_available(self.instance) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 726 | |
Anas Nashif | 2a74053 | 2021-02-04 08:08:20 -0500 | [diff] [blame] | 727 | runner = hardware.runner or self.suite.west_runner |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 728 | serial_pty = hardware.serial_pty |
Anas Nashif | 2a74053 | 2021-02-04 08:08:20 -0500 | [diff] [blame] | 729 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 730 | ser_pty_process = None |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 731 | if serial_pty: |
| 732 | master, slave = pty.openpty() |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 733 | try: |
Andrei Emeltchenko | 9f7a903 | 2020-09-02 11:43:07 +0300 | [diff] [blame] | 734 | ser_pty_process = subprocess.Popen(re.split(',| ', serial_pty), stdout=master, stdin=master, stderr=master) |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 735 | except subprocess.CalledProcessError as error: |
| 736 | logger.error("Failed to run subprocess {}, error {}".format(serial_pty, error.output)) |
| 737 | return |
| 738 | |
| 739 | serial_device = os.ttyname(slave) |
| 740 | else: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 741 | serial_device = hardware.serial |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 742 | |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 743 | logger.debug("Using serial device {} @ {} baud".format(serial_device, hardware.serial_baud)) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 744 | |
Øyvind Rønningstad | f72aef1 | 2020-07-01 16:58:54 +0200 | [diff] [blame] | 745 | if (self.suite.west_flash is not None) or runner: |
| 746 | command = ["west", "flash", "--skip-rebuild", "-d", self.build_dir] |
| 747 | command_extra_args = [] |
| 748 | |
| 749 | # There are three ways this option is used. |
| 750 | # 1) bare: --west-flash |
| 751 | # This results in options.west_flash == [] |
| 752 | # 2) with a value: --west-flash="--board-id=42" |
| 753 | # This results in options.west_flash == "--board-id=42" |
| 754 | # 3) Multiple values: --west-flash="--board-id=42,--erase" |
| 755 | # This results in options.west_flash == "--board-id=42 --erase" |
| 756 | if self.suite.west_flash and self.suite.west_flash != []: |
| 757 | command_extra_args.extend(self.suite.west_flash.split(',')) |
| 758 | |
| 759 | if runner: |
| 760 | command.append("--runner") |
| 761 | command.append(runner) |
| 762 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 763 | board_id = hardware.probe_id or hardware.id |
| 764 | product = hardware.product |
Øyvind Rønningstad | f72aef1 | 2020-07-01 16:58:54 +0200 | [diff] [blame] | 765 | if board_id is not None: |
| 766 | if runner == "pyocd": |
| 767 | command_extra_args.append("--board-id") |
| 768 | command_extra_args.append(board_id) |
| 769 | elif runner == "nrfjprog": |
| 770 | command_extra_args.append("--snr") |
| 771 | command_extra_args.append(board_id) |
| 772 | elif runner == "openocd" and product == "STM32 STLink": |
| 773 | command_extra_args.append("--cmd-pre-init") |
| 774 | command_extra_args.append("hla_serial %s" % (board_id)) |
| 775 | elif runner == "openocd" and product == "STLINK-V3": |
| 776 | command_extra_args.append("--cmd-pre-init") |
| 777 | command_extra_args.append("hla_serial %s" % (board_id)) |
| 778 | elif runner == "openocd" and product == "EDBG CMSIS-DAP": |
| 779 | command_extra_args.append("--cmd-pre-init") |
| 780 | command_extra_args.append("cmsis_dap_serial %s" % (board_id)) |
| 781 | elif runner == "jlink": |
| 782 | command.append("--tool-opt=-SelectEmuBySN %s" % (board_id)) |
| 783 | |
| 784 | if command_extra_args != []: |
| 785 | command.append('--') |
| 786 | command.extend(command_extra_args) |
| 787 | else: |
| 788 | command = [self.generator_cmd, "-C", self.build_dir, "flash"] |
| 789 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 790 | pre_script = hardware.pre_script |
| 791 | post_flash_script = hardware.post_flash_script |
| 792 | post_script = hardware.post_script |
Watson Zeng | 3b43d94 | 2020-08-25 15:22:39 +0800 | [diff] [blame] | 793 | |
| 794 | if pre_script: |
| 795 | self.run_custom_script(pre_script, 30) |
| 796 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 797 | try: |
| 798 | ser = serial.Serial( |
| 799 | serial_device, |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 800 | baudrate=hardware.serial_baud, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 801 | parity=serial.PARITY_NONE, |
| 802 | stopbits=serial.STOPBITS_ONE, |
| 803 | bytesize=serial.EIGHTBITS, |
| 804 | timeout=self.timeout |
| 805 | ) |
| 806 | except serial.SerialException as e: |
| 807 | self.set_state("failed", 0) |
| 808 | self.instance.reason = "Failed" |
| 809 | logger.error("Serial device error: %s" % (str(e))) |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 810 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 811 | if serial_pty and ser_pty_process: |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 812 | ser_pty_process.terminate() |
| 813 | outs, errs = ser_pty_process.communicate() |
| 814 | logger.debug("Process {} terminated outs: {} errs {}".format(serial_pty, outs, errs)) |
| 815 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 816 | self.make_device_available(serial_device) |
| 817 | return |
| 818 | |
| 819 | ser.flush() |
| 820 | |
| 821 | harness_name = self.instance.testcase.harness.capitalize() |
| 822 | harness_import = HarnessImporter(harness_name) |
| 823 | harness = harness_import.instance |
| 824 | harness.configure(self.instance) |
| 825 | read_pipe, write_pipe = os.pipe() |
| 826 | start_time = time.time() |
| 827 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 828 | t = threading.Thread(target=self.monitor_serial, daemon=True, |
| 829 | args=(ser, read_pipe, harness)) |
| 830 | t.start() |
| 831 | |
| 832 | d_log = "{}/device.log".format(self.instance.build_dir) |
| 833 | logger.debug('Flash command: %s', command) |
| 834 | try: |
| 835 | stdout = stderr = None |
| 836 | with subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as proc: |
| 837 | try: |
| 838 | (stdout, stderr) = proc.communicate(timeout=30) |
| 839 | logger.debug(stdout.decode()) |
| 840 | |
| 841 | if proc.returncode != 0: |
| 842 | self.instance.reason = "Device issue (Flash?)" |
| 843 | with open(d_log, "w") as dlog_fp: |
| 844 | dlog_fp.write(stderr.decode()) |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 845 | os.write(write_pipe, b'x') # halt the thread |
| 846 | out_state = "flash_error" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 847 | except subprocess.TimeoutExpired: |
| 848 | proc.kill() |
| 849 | (stdout, stderr) = proc.communicate() |
| 850 | self.instance.reason = "Device issue (Timeout)" |
| 851 | |
| 852 | with open(d_log, "w") as dlog_fp: |
| 853 | dlog_fp.write(stderr.decode()) |
| 854 | |
| 855 | except subprocess.CalledProcessError: |
| 856 | os.write(write_pipe, b'x') # halt the thread |
| 857 | |
| 858 | if post_flash_script: |
| 859 | self.run_custom_script(post_flash_script, 30) |
| 860 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 861 | t.join(self.timeout) |
| 862 | if t.is_alive(): |
| 863 | logger.debug("Timed out while monitoring serial output on {}".format(self.instance.platform.name)) |
| 864 | out_state = "timeout" |
| 865 | |
| 866 | if ser.isOpen(): |
| 867 | ser.close() |
| 868 | |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 869 | if serial_pty: |
| 870 | ser_pty_process.terminate() |
| 871 | outs, errs = ser_pty_process.communicate() |
| 872 | logger.debug("Process {} terminated outs: {} errs {}".format(serial_pty, outs, errs)) |
| 873 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 874 | os.close(write_pipe) |
| 875 | os.close(read_pipe) |
| 876 | |
| 877 | handler_time = time.time() - start_time |
| 878 | |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 879 | if out_state in ["timeout", "flash_error"]: |
Piotr Golyzniak | 0e1ecb3 | 2021-09-23 00:14:04 +0200 | [diff] [blame] | 880 | self.add_missing_testscases(harness) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 881 | |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 882 | if out_state == "timeout": |
| 883 | self.instance.reason = "Timeout" |
| 884 | elif out_state == "flash_error": |
| 885 | self.instance.reason = "Flash error" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 886 | |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 887 | if harness.is_pytest: |
| 888 | harness.pytest_run(self.log) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 889 | self.instance.results = harness.tests |
| 890 | |
peng1 chen | 6478735 | 2021-01-27 11:00:05 +0800 | [diff] [blame] | 891 | # sometimes a test instance hasn't been executed successfully with an |
| 892 | # empty dictionary results, in order to include it into final report, |
| 893 | # so fill the results as BLOCK |
| 894 | if self.instance.results == {}: |
| 895 | for k in self.instance.testcase.cases: |
| 896 | self.instance.results[k] = 'BLOCK' |
| 897 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 898 | if harness.state: |
| 899 | self.set_state(harness.state, handler_time) |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 900 | if harness.state == "failed": |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 901 | self.instance.reason = "Failed" |
| 902 | else: |
| 903 | self.set_state(out_state, handler_time) |
| 904 | |
| 905 | if post_script: |
| 906 | self.run_custom_script(post_script, 30) |
| 907 | |
| 908 | self.make_device_available(serial_device) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 909 | self.record(harness) |
| 910 | |
| 911 | |
| 912 | class QEMUHandler(Handler): |
| 913 | """Spawns a thread to monitor QEMU output from pipes |
| 914 | |
| 915 | We pass QEMU_PIPE to 'make run' and monitor the pipes for output. |
| 916 | We need to do this as once qemu starts, it runs forever until killed. |
| 917 | Test cases emit special messages to the console as they run, we check |
| 918 | for these to collect whether the test passed or failed. |
| 919 | """ |
| 920 | |
| 921 | def __init__(self, instance, type_str): |
| 922 | """Constructor |
| 923 | |
| 924 | @param instance Test instance |
| 925 | """ |
| 926 | |
| 927 | super().__init__(instance, type_str) |
| 928 | self.fifo_fn = os.path.join(instance.build_dir, "qemu-fifo") |
| 929 | |
| 930 | self.pid_fn = os.path.join(instance.build_dir, "qemu.pid") |
| 931 | |
Daniel Leung | faae15d | 2020-08-18 10:13:35 -0700 | [diff] [blame] | 932 | if "ignore_qemu_crash" in instance.testcase.tags: |
| 933 | self.ignore_qemu_crash = True |
| 934 | self.ignore_unexpected_eof = True |
| 935 | else: |
| 936 | self.ignore_qemu_crash = False |
| 937 | self.ignore_unexpected_eof = False |
| 938 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 939 | @staticmethod |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 940 | def _get_cpu_time(pid): |
| 941 | """get process CPU time. |
| 942 | |
| 943 | The guest virtual time in QEMU icount mode isn't host time and |
| 944 | it's maintained by counting guest instructions, so we use QEMU |
| 945 | process exection time to mostly simulate the time of guest OS. |
| 946 | """ |
| 947 | proc = psutil.Process(pid) |
| 948 | cpu_time = proc.cpu_times() |
| 949 | return cpu_time.user + cpu_time.system |
| 950 | |
| 951 | @staticmethod |
Daniel Leung | faae15d | 2020-08-18 10:13:35 -0700 | [diff] [blame] | 952 | def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results, harness, |
| 953 | ignore_unexpected_eof=False): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 954 | fifo_in = fifo_fn + ".in" |
| 955 | fifo_out = fifo_fn + ".out" |
| 956 | |
| 957 | # These in/out nodes are named from QEMU's perspective, not ours |
| 958 | if os.path.exists(fifo_in): |
| 959 | os.unlink(fifo_in) |
| 960 | os.mkfifo(fifo_in) |
| 961 | if os.path.exists(fifo_out): |
| 962 | os.unlink(fifo_out) |
| 963 | os.mkfifo(fifo_out) |
| 964 | |
| 965 | # We don't do anything with out_fp but we need to open it for |
| 966 | # writing so that QEMU doesn't block, due to the way pipes work |
| 967 | out_fp = open(fifo_in, "wb") |
| 968 | # Disable internal buffering, we don't |
| 969 | # want read() or poll() to ever block if there is data in there |
| 970 | in_fp = open(fifo_out, "rb", buffering=0) |
| 971 | log_out_fp = open(logfile, "wt") |
| 972 | |
| 973 | start_time = time.time() |
| 974 | timeout_time = start_time + timeout |
| 975 | p = select.poll() |
| 976 | p.register(in_fp, select.POLLIN) |
| 977 | out_state = None |
| 978 | |
| 979 | line = "" |
| 980 | timeout_extended = False |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 981 | |
| 982 | pid = 0 |
| 983 | if os.path.exists(pid_fn): |
| 984 | pid = int(open(pid_fn).read()) |
| 985 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 986 | while True: |
| 987 | this_timeout = int((timeout_time - time.time()) * 1000) |
| 988 | if this_timeout < 0 or not p.poll(this_timeout): |
Wentong Wu | 517633c | 2020-07-24 21:13:01 +0800 | [diff] [blame] | 989 | try: |
| 990 | if pid and this_timeout > 0: |
| 991 | #there's possibility we polled nothing because |
| 992 | #of not enough CPU time scheduled by host for |
| 993 | #QEMU process during p.poll(this_timeout) |
| 994 | cpu_time = QEMUHandler._get_cpu_time(pid) |
| 995 | if cpu_time < timeout and not out_state: |
| 996 | timeout_time = time.time() + (timeout - cpu_time) |
| 997 | continue |
| 998 | except ProcessLookupError: |
| 999 | out_state = "failed" |
| 1000 | break |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 1001 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1002 | if not out_state: |
| 1003 | out_state = "timeout" |
| 1004 | break |
| 1005 | |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 1006 | if pid == 0 and os.path.exists(pid_fn): |
| 1007 | pid = int(open(pid_fn).read()) |
| 1008 | |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 1009 | if harness.is_pytest: |
| 1010 | harness.handle(None) |
| 1011 | out_state = harness.state |
| 1012 | break |
| 1013 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1014 | try: |
| 1015 | c = in_fp.read(1).decode("utf-8") |
| 1016 | except UnicodeDecodeError: |
| 1017 | # Test is writing something weird, fail |
| 1018 | out_state = "unexpected byte" |
| 1019 | break |
| 1020 | |
| 1021 | if c == "": |
| 1022 | # EOF, this shouldn't happen unless QEMU crashes |
Daniel Leung | faae15d | 2020-08-18 10:13:35 -0700 | [diff] [blame] | 1023 | if not ignore_unexpected_eof: |
| 1024 | out_state = "unexpected eof" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1025 | break |
| 1026 | line = line + c |
| 1027 | if c != "\n": |
| 1028 | continue |
| 1029 | |
| 1030 | # line contains a full line of data output from QEMU |
| 1031 | log_out_fp.write(line) |
| 1032 | log_out_fp.flush() |
| 1033 | line = line.strip() |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1034 | logger.debug(f"QEMU ({pid}): {line}") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1035 | |
| 1036 | harness.handle(line) |
| 1037 | if harness.state: |
| 1038 | # if we have registered a fail make sure the state is not |
| 1039 | # overridden by a false success message coming from the |
| 1040 | # testsuite |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 1041 | if out_state not in ['failed', 'unexpected eof', 'unexpected byte']: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1042 | out_state = harness.state |
| 1043 | |
| 1044 | # if we get some state, that means test is doing well, we reset |
| 1045 | # the timeout and wait for 2 more seconds to catch anything |
| 1046 | # printed late. We wait much longer if code |
| 1047 | # coverage is enabled since dumping this information can |
| 1048 | # take some time. |
| 1049 | if not timeout_extended or harness.capture_coverage: |
| 1050 | timeout_extended = True |
| 1051 | if harness.capture_coverage: |
| 1052 | timeout_time = time.time() + 30 |
| 1053 | else: |
| 1054 | timeout_time = time.time() + 2 |
| 1055 | line = "" |
| 1056 | |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 1057 | if harness.is_pytest: |
| 1058 | harness.pytest_run(logfile) |
| 1059 | out_state = harness.state |
| 1060 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1061 | handler.record(harness) |
| 1062 | |
| 1063 | handler_time = time.time() - start_time |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1064 | logger.debug(f"QEMU ({pid}) complete ({out_state}) after {handler_time} seconds") |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 1065 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1066 | if out_state == "timeout": |
| 1067 | handler.instance.reason = "Timeout" |
Anas Nashif | 0605292 | 2020-07-15 22:44:24 -0400 | [diff] [blame] | 1068 | handler.set_state("failed", handler_time) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1069 | elif out_state == "failed": |
| 1070 | handler.instance.reason = "Failed" |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 1071 | handler.set_state("failed", handler_time) |
Anas Nashif | 0605292 | 2020-07-15 22:44:24 -0400 | [diff] [blame] | 1072 | elif out_state in ['unexpected eof', 'unexpected byte']: |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 1073 | handler.instance.reason = out_state |
Anas Nashif | 0605292 | 2020-07-15 22:44:24 -0400 | [diff] [blame] | 1074 | handler.set_state("failed", handler_time) |
| 1075 | else: |
| 1076 | handler.set_state(out_state, handler_time) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1077 | |
| 1078 | log_out_fp.close() |
| 1079 | out_fp.close() |
| 1080 | in_fp.close() |
Wentong Wu | 0d619ae | 2020-05-05 19:46:49 -0400 | [diff] [blame] | 1081 | if pid: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1082 | try: |
| 1083 | if pid: |
| 1084 | os.kill(pid, signal.SIGTERM) |
| 1085 | except ProcessLookupError: |
| 1086 | # Oh well, as long as it's dead! User probably sent Ctrl-C |
| 1087 | pass |
| 1088 | |
| 1089 | os.unlink(fifo_in) |
| 1090 | os.unlink(fifo_out) |
| 1091 | |
| 1092 | def handle(self): |
| 1093 | self.results = {} |
| 1094 | self.run = True |
| 1095 | |
| 1096 | # We pass this to QEMU which looks for fifos with .in and .out |
| 1097 | # suffixes. |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1098 | |
Anas Nashif | c1c1099 | 2020-09-10 07:36:00 -0400 | [diff] [blame] | 1099 | self.fifo_fn = os.path.join(self.instance.build_dir, "qemu-fifo") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1100 | self.pid_fn = os.path.join(self.instance.build_dir, "qemu.pid") |
Anas Nashif | c1c1099 | 2020-09-10 07:36:00 -0400 | [diff] [blame] | 1101 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1102 | if os.path.exists(self.pid_fn): |
| 1103 | os.unlink(self.pid_fn) |
| 1104 | |
| 1105 | self.log_fn = self.log |
| 1106 | |
| 1107 | harness_import = HarnessImporter(self.instance.testcase.harness.capitalize()) |
| 1108 | harness = harness_import.instance |
| 1109 | harness.configure(self.instance) |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 1110 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1111 | self.thread = threading.Thread(name=self.name, target=QEMUHandler._thread, |
| 1112 | args=(self, self.timeout, self.build_dir, |
| 1113 | self.log_fn, self.fifo_fn, |
Daniel Leung | faae15d | 2020-08-18 10:13:35 -0700 | [diff] [blame] | 1114 | self.pid_fn, self.results, harness, |
| 1115 | self.ignore_unexpected_eof)) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1116 | |
| 1117 | self.instance.results = harness.tests |
| 1118 | self.thread.daemon = True |
| 1119 | logger.debug("Spawning QEMUHandler Thread for %s" % self.name) |
| 1120 | self.thread.start() |
Eugeniy Paltsev | afb34e3 | 2020-11-29 21:28:28 +0300 | [diff] [blame] | 1121 | if sys.stdout.isatty(): |
| 1122 | subprocess.call(["stty", "sane"]) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1123 | |
| 1124 | logger.debug("Running %s (%s)" % (self.name, self.type_str)) |
| 1125 | command = [self.generator_cmd] |
| 1126 | command += ["-C", self.build_dir, "run"] |
| 1127 | |
Anas Nashif | fdc02b6 | 2020-09-09 19:11:19 -0400 | [diff] [blame] | 1128 | is_timeout = False |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1129 | qemu_pid = None |
Anas Nashif | fdc02b6 | 2020-09-09 19:11:19 -0400 | [diff] [blame] | 1130 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1131 | with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.build_dir) as proc: |
| 1132 | logger.debug("Spawning QEMUHandler Thread for %s" % self.name) |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1133 | |
Wentong Wu | 7ec57b4 | 2020-05-05 19:19:18 -0400 | [diff] [blame] | 1134 | try: |
| 1135 | proc.wait(self.timeout) |
| 1136 | except subprocess.TimeoutExpired: |
Anas Nashif | fdc02b6 | 2020-09-09 19:11:19 -0400 | [diff] [blame] | 1137 | # sometimes QEMU can't handle SIGTERM signal correctly |
| 1138 | # in that case kill -9 QEMU process directly and leave |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 1139 | # twister to judge testing result by console output |
Anas Nashif | fdc02b6 | 2020-09-09 19:11:19 -0400 | [diff] [blame] | 1140 | |
| 1141 | is_timeout = True |
Jingru Wang | fed1c54 | 2021-06-11 11:54:55 +0800 | [diff] [blame] | 1142 | self.terminate(proc) |
| 1143 | if harness.state == "passed": |
| 1144 | self.returncode = 0 |
Wentong Wu | 7ec57b4 | 2020-05-05 19:19:18 -0400 | [diff] [blame] | 1145 | else: |
Wentong Wu | 7ec57b4 | 2020-05-05 19:19:18 -0400 | [diff] [blame] | 1146 | self.returncode = proc.returncode |
| 1147 | else: |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1148 | if os.path.exists(self.pid_fn): |
| 1149 | qemu_pid = int(open(self.pid_fn).read()) |
| 1150 | logger.debug(f"No timeout, return code from QEMU ({qemu_pid}): {proc.returncode}") |
Wentong Wu | 7ec57b4 | 2020-05-05 19:19:18 -0400 | [diff] [blame] | 1151 | self.returncode = proc.returncode |
Daniel Leung | 5b1b4a3 | 2020-08-18 10:10:36 -0700 | [diff] [blame] | 1152 | # Need to wait for harness to finish processing |
| 1153 | # output from QEMU. Otherwise it might miss some |
| 1154 | # error messages. |
Jingru Wang | fed1c54 | 2021-06-11 11:54:55 +0800 | [diff] [blame] | 1155 | self.thread.join(0) |
| 1156 | if self.thread.is_alive(): |
| 1157 | logger.debug("Timed out while monitoring QEMU output") |
Daniel Leung | 5b1b4a3 | 2020-08-18 10:10:36 -0700 | [diff] [blame] | 1158 | |
Wentong Wu | 7ec57b4 | 2020-05-05 19:19:18 -0400 | [diff] [blame] | 1159 | if os.path.exists(self.pid_fn): |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1160 | qemu_pid = int(open(self.pid_fn).read()) |
Wentong Wu | 7ec57b4 | 2020-05-05 19:19:18 -0400 | [diff] [blame] | 1161 | os.unlink(self.pid_fn) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1162 | |
Anas Nashif | 743594f | 2020-09-10 08:19:47 -0400 | [diff] [blame] | 1163 | logger.debug(f"return code from QEMU ({qemu_pid}): {self.returncode}") |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 1164 | |
Daniel Leung | faae15d | 2020-08-18 10:13:35 -0700 | [diff] [blame] | 1165 | if (self.returncode != 0 and not self.ignore_qemu_crash) or not harness.state: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1166 | self.set_state("failed", 0) |
Anas Nashif | fdc02b6 | 2020-09-09 19:11:19 -0400 | [diff] [blame] | 1167 | if is_timeout: |
| 1168 | self.instance.reason = "Timeout" |
| 1169 | else: |
| 1170 | self.instance.reason = "Exited with {}".format(self.returncode) |
Piotr Golyzniak | 0e1ecb3 | 2021-09-23 00:14:04 +0200 | [diff] [blame] | 1171 | self.add_missing_testscases(harness) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1172 | |
| 1173 | def get_fifo(self): |
| 1174 | return self.fifo_fn |
| 1175 | |
| 1176 | |
| 1177 | class SizeCalculator: |
| 1178 | alloc_sections = [ |
| 1179 | "bss", |
| 1180 | "noinit", |
| 1181 | "app_bss", |
| 1182 | "app_noinit", |
| 1183 | "ccm_bss", |
| 1184 | "ccm_noinit" |
| 1185 | ] |
| 1186 | |
| 1187 | rw_sections = [ |
| 1188 | "datas", |
| 1189 | "initlevel", |
| 1190 | "exceptions", |
| 1191 | "initshell", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1192 | "_static_thread_data_area", |
| 1193 | "k_timer_area", |
| 1194 | "k_mem_slab_area", |
| 1195 | "k_mem_pool_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1196 | "sw_isr_table", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1197 | "k_sem_area", |
| 1198 | "k_mutex_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1199 | "app_shmem_regions", |
| 1200 | "_k_fifo_area", |
| 1201 | "_k_lifo_area", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1202 | "k_stack_area", |
| 1203 | "k_msgq_area", |
| 1204 | "k_mbox_area", |
| 1205 | "k_pipe_area", |
Daniel Leung | 203556c | 2020-08-24 12:40:26 -0700 | [diff] [blame] | 1206 | "net_if_area", |
| 1207 | "net_if_dev_area", |
| 1208 | "net_l2_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1209 | "net_l2_data", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1210 | "k_queue_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1211 | "_net_buf_pool_area", |
| 1212 | "app_datas", |
| 1213 | "kobject_data", |
| 1214 | "mmu_tables", |
| 1215 | "app_pad", |
| 1216 | "priv_stacks", |
| 1217 | "ccm_data", |
| 1218 | "usb_descriptor", |
| 1219 | "usb_data", "usb_bos_desc", |
Jukka Rissanen | 420b195 | 2020-04-01 12:47:53 +0300 | [diff] [blame] | 1220 | "uart_mux", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1221 | 'log_backends_sections', |
| 1222 | 'log_dynamic_sections', |
| 1223 | 'log_const_sections', |
| 1224 | "app_smem", |
| 1225 | 'shell_root_cmds_sections', |
| 1226 | 'log_const_sections', |
| 1227 | "font_entry_sections", |
| 1228 | "priv_stacks_noinit", |
| 1229 | "_GCOV_BSS_SECTION_NAME", |
| 1230 | "gcov", |
Daniel Leung | 203556c | 2020-08-24 12:40:26 -0700 | [diff] [blame] | 1231 | "nocache", |
| 1232 | "devices", |
| 1233 | "k_heap_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1234 | ] |
| 1235 | |
| 1236 | # These get copied into RAM only on non-XIP |
| 1237 | ro_sections = [ |
| 1238 | "rom_start", |
| 1239 | "text", |
| 1240 | "ctors", |
| 1241 | "init_array", |
| 1242 | "reset", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1243 | "z_object_assignment_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1244 | "rodata", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1245 | "net_l2", |
| 1246 | "vector", |
| 1247 | "sw_isr_table", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1248 | "settings_handler_static_area", |
Daniel Leung | 203556c | 2020-08-24 12:40:26 -0700 | [diff] [blame] | 1249 | "bt_l2cap_fixed_chan_area", |
| 1250 | "bt_l2cap_br_fixed_chan_area", |
| 1251 | "bt_gatt_service_static_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1252 | "vectors", |
Andrew Boie | 45979da | 2020-05-23 14:38:39 -0700 | [diff] [blame] | 1253 | "net_socket_register_area", |
| 1254 | "net_ppp_proto", |
| 1255 | "shell_area", |
| 1256 | "tracing_backend_area", |
Daniel Leung | 203556c | 2020-08-24 12:40:26 -0700 | [diff] [blame] | 1257 | "ppp_protocol_handler_area", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1258 | ] |
| 1259 | |
| 1260 | def __init__(self, filename, extra_sections): |
| 1261 | """Constructor |
| 1262 | |
| 1263 | @param filename Path to the output binary |
| 1264 | The <filename> is parsed by objdump to determine section sizes |
| 1265 | """ |
| 1266 | # Make sure this is an ELF binary |
| 1267 | with open(filename, "rb") as f: |
| 1268 | magic = f.read(4) |
| 1269 | |
| 1270 | try: |
| 1271 | if magic != b'\x7fELF': |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 1272 | raise TwisterRuntimeError("%s is not an ELF binary" % filename) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1273 | except Exception as e: |
| 1274 | print(str(e)) |
| 1275 | sys.exit(2) |
| 1276 | |
| 1277 | # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK. |
| 1278 | # GREP can not be used as it returns an error if the symbol is not |
| 1279 | # found. |
| 1280 | is_xip_command = "nm " + filename + \ |
| 1281 | " | awk '/CONFIG_XIP/ { print $3 }'" |
| 1282 | is_xip_output = subprocess.check_output( |
| 1283 | is_xip_command, shell=True, stderr=subprocess.STDOUT).decode( |
| 1284 | "utf-8").strip() |
| 1285 | try: |
| 1286 | if is_xip_output.endswith("no symbols"): |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 1287 | raise TwisterRuntimeError("%s has no symbol information" % filename) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1288 | except Exception as e: |
| 1289 | print(str(e)) |
| 1290 | sys.exit(2) |
| 1291 | |
| 1292 | self.is_xip = (len(is_xip_output) != 0) |
| 1293 | |
| 1294 | self.filename = filename |
| 1295 | self.sections = [] |
| 1296 | self.rom_size = 0 |
| 1297 | self.ram_size = 0 |
| 1298 | self.extra_sections = extra_sections |
| 1299 | |
| 1300 | self._calculate_sizes() |
| 1301 | |
| 1302 | def get_ram_size(self): |
| 1303 | """Get the amount of RAM the application will use up on the device |
| 1304 | |
| 1305 | @return amount of RAM, in bytes |
| 1306 | """ |
| 1307 | return self.ram_size |
| 1308 | |
| 1309 | def get_rom_size(self): |
| 1310 | """Get the size of the data that this application uses on device's flash |
| 1311 | |
| 1312 | @return amount of ROM, in bytes |
| 1313 | """ |
| 1314 | return self.rom_size |
| 1315 | |
| 1316 | def unrecognized_sections(self): |
| 1317 | """Get a list of sections inside the binary that weren't recognized |
| 1318 | |
| 1319 | @return list of unrecognized section names |
| 1320 | """ |
| 1321 | slist = [] |
| 1322 | for v in self.sections: |
| 1323 | if not v["recognized"]: |
| 1324 | slist.append(v["name"]) |
| 1325 | return slist |
| 1326 | |
| 1327 | def _calculate_sizes(self): |
| 1328 | """ Calculate RAM and ROM usage by section """ |
| 1329 | objdump_command = "objdump -h " + self.filename |
| 1330 | objdump_output = subprocess.check_output( |
| 1331 | objdump_command, shell=True).decode("utf-8").splitlines() |
| 1332 | |
| 1333 | for line in objdump_output: |
| 1334 | words = line.split() |
| 1335 | |
| 1336 | if not words: # Skip lines that are too short |
| 1337 | continue |
| 1338 | |
| 1339 | index = words[0] |
| 1340 | if not index[0].isdigit(): # Skip lines that do not start |
| 1341 | continue # with a digit |
| 1342 | |
| 1343 | name = words[1] # Skip lines with section names |
| 1344 | if name[0] == '.': # starting with '.' |
| 1345 | continue |
| 1346 | |
| 1347 | # TODO this doesn't actually reflect the size in flash or RAM as |
| 1348 | # it doesn't include linker-imposed padding between sections. |
| 1349 | # It is close though. |
| 1350 | size = int(words[2], 16) |
| 1351 | if size == 0: |
| 1352 | continue |
| 1353 | |
| 1354 | load_addr = int(words[4], 16) |
| 1355 | virt_addr = int(words[3], 16) |
| 1356 | |
| 1357 | # Add section to memory use totals (for both non-XIP and XIP scenarios) |
| 1358 | # Unrecognized section names are not included in the calculations. |
| 1359 | recognized = True |
| 1360 | if name in SizeCalculator.alloc_sections: |
| 1361 | self.ram_size += size |
| 1362 | stype = "alloc" |
| 1363 | elif name in SizeCalculator.rw_sections: |
| 1364 | self.ram_size += size |
| 1365 | self.rom_size += size |
| 1366 | stype = "rw" |
| 1367 | elif name in SizeCalculator.ro_sections: |
| 1368 | self.rom_size += size |
| 1369 | if not self.is_xip: |
| 1370 | self.ram_size += size |
| 1371 | stype = "ro" |
| 1372 | else: |
| 1373 | stype = "unknown" |
| 1374 | if name not in self.extra_sections: |
| 1375 | recognized = False |
| 1376 | |
| 1377 | self.sections.append({"name": name, "load_addr": load_addr, |
| 1378 | "size": size, "virt_addr": virt_addr, |
| 1379 | "type": stype, "recognized": recognized}) |
| 1380 | |
| 1381 | |
| 1382 | |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 1383 | class TwisterConfigParser: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1384 | """Class to read test case files with semantic checking |
| 1385 | """ |
| 1386 | |
| 1387 | def __init__(self, filename, schema): |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 1388 | """Instantiate a new TwisterConfigParser object |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1389 | |
| 1390 | @param filename Source .yaml file to read |
| 1391 | """ |
| 1392 | self.data = {} |
| 1393 | self.schema = schema |
| 1394 | self.filename = filename |
| 1395 | self.tests = {} |
| 1396 | self.common = {} |
| 1397 | |
| 1398 | def load(self): |
| 1399 | self.data = scl.yaml_load_verify(self.filename, self.schema) |
| 1400 | |
| 1401 | if 'tests' in self.data: |
| 1402 | self.tests = self.data['tests'] |
| 1403 | if 'common' in self.data: |
| 1404 | self.common = self.data['common'] |
| 1405 | |
| 1406 | def _cast_value(self, value, typestr): |
| 1407 | if isinstance(value, str): |
| 1408 | v = value.strip() |
| 1409 | if typestr == "str": |
| 1410 | return v |
| 1411 | |
| 1412 | elif typestr == "float": |
| 1413 | return float(value) |
| 1414 | |
| 1415 | elif typestr == "int": |
| 1416 | return int(value) |
| 1417 | |
| 1418 | elif typestr == "bool": |
| 1419 | return value |
| 1420 | |
| 1421 | elif typestr.startswith("list") and isinstance(value, list): |
| 1422 | return value |
| 1423 | elif typestr.startswith("list") and isinstance(value, str): |
| 1424 | vs = v.split() |
| 1425 | if len(typestr) > 4 and typestr[4] == ":": |
| 1426 | return [self._cast_value(vsi, typestr[5:]) for vsi in vs] |
| 1427 | else: |
| 1428 | return vs |
| 1429 | |
| 1430 | elif typestr.startswith("set"): |
| 1431 | vs = v.split() |
| 1432 | if len(typestr) > 3 and typestr[3] == ":": |
| 1433 | return {self._cast_value(vsi, typestr[4:]) for vsi in vs} |
| 1434 | else: |
| 1435 | return set(vs) |
| 1436 | |
| 1437 | elif typestr.startswith("map"): |
| 1438 | return value |
| 1439 | else: |
| 1440 | raise ConfigurationError( |
| 1441 | self.filename, "unknown type '%s'" % value) |
| 1442 | |
| 1443 | def get_test(self, name, valid_keys): |
| 1444 | """Get a dictionary representing the keys/values within a test |
| 1445 | |
| 1446 | @param name The test in the .yaml file to retrieve data from |
| 1447 | @param valid_keys A dictionary representing the intended semantics |
| 1448 | for this test. Each key in this dictionary is a key that could |
| 1449 | be specified, if a key is given in the .yaml file which isn't in |
| 1450 | here, it will generate an error. Each value in this dictionary |
| 1451 | is another dictionary containing metadata: |
| 1452 | |
| 1453 | "default" - Default value if not given |
| 1454 | "type" - Data type to convert the text value to. Simple types |
| 1455 | supported are "str", "float", "int", "bool" which will get |
| 1456 | converted to respective Python data types. "set" and "list" |
| 1457 | may also be specified which will split the value by |
| 1458 | whitespace (but keep the elements as strings). finally, |
| 1459 | "list:<type>" and "set:<type>" may be given which will |
| 1460 | perform a type conversion after splitting the value up. |
| 1461 | "required" - If true, raise an error if not defined. If false |
| 1462 | and "default" isn't specified, a type conversion will be |
| 1463 | done on an empty string |
| 1464 | @return A dictionary containing the test key-value pairs with |
| 1465 | type conversion and default values filled in per valid_keys |
| 1466 | """ |
| 1467 | |
| 1468 | d = {} |
| 1469 | for k, v in self.common.items(): |
| 1470 | d[k] = v |
| 1471 | |
| 1472 | for k, v in self.tests[name].items(): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1473 | if k in d: |
| 1474 | if isinstance(d[k], str): |
| 1475 | # By default, we just concatenate string values of keys |
| 1476 | # which appear both in "common" and per-test sections, |
| 1477 | # but some keys are handled in adhoc way based on their |
| 1478 | # semantics. |
| 1479 | if k == "filter": |
| 1480 | d[k] = "(%s) and (%s)" % (d[k], v) |
| 1481 | else: |
| 1482 | d[k] += " " + v |
| 1483 | else: |
| 1484 | d[k] = v |
| 1485 | |
| 1486 | for k, kinfo in valid_keys.items(): |
| 1487 | if k not in d: |
| 1488 | if "required" in kinfo: |
| 1489 | required = kinfo["required"] |
| 1490 | else: |
| 1491 | required = False |
| 1492 | |
| 1493 | if required: |
| 1494 | raise ConfigurationError( |
| 1495 | self.filename, |
| 1496 | "missing required value for '%s' in test '%s'" % |
| 1497 | (k, name)) |
| 1498 | else: |
| 1499 | if "default" in kinfo: |
| 1500 | default = kinfo["default"] |
| 1501 | else: |
| 1502 | default = self._cast_value("", kinfo["type"]) |
| 1503 | d[k] = default |
| 1504 | else: |
| 1505 | try: |
| 1506 | d[k] = self._cast_value(d[k], kinfo["type"]) |
| 1507 | except ValueError: |
| 1508 | raise ConfigurationError( |
| 1509 | self.filename, "bad %s value '%s' for key '%s' in name '%s'" % |
| 1510 | (kinfo["type"], d[k], k, name)) |
| 1511 | |
| 1512 | return d |
| 1513 | |
| 1514 | |
| 1515 | class Platform: |
| 1516 | """Class representing metadata for a particular platform |
| 1517 | |
| 1518 | Maps directly to BOARD when building""" |
| 1519 | |
| 1520 | platform_schema = scl.yaml_load(os.path.join(ZEPHYR_BASE, |
Anas Nashif | c24bf6f | 2020-12-07 11:18:07 -0500 | [diff] [blame] | 1521 | "scripts", "schemas", "twister", "platform-schema.yaml")) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1522 | |
| 1523 | def __init__(self): |
| 1524 | """Constructor. |
| 1525 | |
| 1526 | """ |
| 1527 | |
| 1528 | self.name = "" |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 1529 | self.twister = True |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1530 | # if no RAM size is specified by the board, take a default of 128K |
| 1531 | self.ram = 128 |
| 1532 | |
| 1533 | self.ignore_tags = [] |
Anas Nashif | e8e367a | 2020-07-16 16:27:04 -0400 | [diff] [blame] | 1534 | self.only_tags = [] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1535 | self.default = False |
| 1536 | # if no flash size is specified by the board, take a default of 512K |
| 1537 | self.flash = 512 |
| 1538 | self.supported = set() |
| 1539 | |
| 1540 | self.arch = "" |
| 1541 | self.type = "na" |
| 1542 | self.simulation = "na" |
| 1543 | self.supported_toolchains = [] |
| 1544 | self.env = [] |
| 1545 | self.env_satisfied = True |
| 1546 | self.filter_data = dict() |
| 1547 | |
| 1548 | def load(self, platform_file): |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 1549 | scp = TwisterConfigParser(platform_file, self.platform_schema) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1550 | scp.load() |
| 1551 | data = scp.data |
| 1552 | |
| 1553 | self.name = data['identifier'] |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 1554 | self.twister = data.get("twister", True) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1555 | # if no RAM size is specified by the board, take a default of 128K |
| 1556 | self.ram = data.get("ram", 128) |
| 1557 | testing = data.get("testing", {}) |
| 1558 | self.ignore_tags = testing.get("ignore_tags", []) |
Anas Nashif | e8e367a | 2020-07-16 16:27:04 -0400 | [diff] [blame] | 1559 | self.only_tags = testing.get("only_tags", []) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1560 | self.default = testing.get("default", False) |
| 1561 | # if no flash size is specified by the board, take a default of 512K |
| 1562 | self.flash = data.get("flash", 512) |
| 1563 | self.supported = set() |
| 1564 | for supp_feature in data.get("supported", []): |
| 1565 | for item in supp_feature.split(":"): |
| 1566 | self.supported.add(item) |
| 1567 | |
| 1568 | self.arch = data['arch'] |
| 1569 | self.type = data.get('type', "na") |
| 1570 | self.simulation = data.get('simulation', "na") |
| 1571 | self.supported_toolchains = data.get("toolchain", []) |
| 1572 | self.env = data.get("env", []) |
| 1573 | self.env_satisfied = True |
| 1574 | for env in self.env: |
| 1575 | if not os.environ.get(env, None): |
| 1576 | self.env_satisfied = False |
| 1577 | |
| 1578 | def __repr__(self): |
| 1579 | return "<%s on %s>" % (self.name, self.arch) |
| 1580 | |
| 1581 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1582 | class DisablePyTestCollectionMixin(object): |
| 1583 | __test__ = False |
| 1584 | |
| 1585 | |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1586 | class ScanPathResult: |
| 1587 | """Result of the TestCase.scan_path function call. |
| 1588 | |
| 1589 | Attributes: |
| 1590 | matches A list of test cases |
| 1591 | warnings A string containing one or more |
| 1592 | warnings to display |
| 1593 | has_registered_test_suites Whether or not the path contained any |
| 1594 | calls to the ztest_register_test_suite |
| 1595 | macro. |
| 1596 | has_run_registered_test_suites Whether or not the path contained at |
| 1597 | least one call to |
| 1598 | ztest_run_registered_test_suites. |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1599 | has_test_main Whether or not the path contains a |
| 1600 | definition of test_main(void) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1601 | """ |
| 1602 | def __init__(self, |
| 1603 | matches: List[str] = None, |
| 1604 | warnings: str = None, |
| 1605 | has_registered_test_suites: bool = False, |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1606 | has_run_registered_test_suites: bool = False, |
| 1607 | has_test_main: bool = False): |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1608 | self.matches = matches |
| 1609 | self.warnings = warnings |
| 1610 | self.has_registered_test_suites = has_registered_test_suites |
| 1611 | self.has_run_registered_test_suites = has_run_registered_test_suites |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1612 | self.has_test_main = has_test_main |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1613 | |
| 1614 | def __eq__(self, other): |
| 1615 | if not isinstance(other, ScanPathResult): |
| 1616 | return False |
| 1617 | return (sorted(self.matches) == sorted(other.matches) and |
| 1618 | self.warnings == other.warnings and |
| 1619 | (self.has_registered_test_suites == |
| 1620 | other.has_registered_test_suites) and |
| 1621 | (self.has_run_registered_test_suites == |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1622 | other.has_run_registered_test_suites) and |
| 1623 | self.has_test_main == other.has_test_main) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1624 | |
| 1625 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1626 | class TestCase(DisablePyTestCollectionMixin): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1627 | """Class representing a test application |
| 1628 | """ |
| 1629 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1630 | def __init__(self, testcase_root, workdir, name): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1631 | """TestCase constructor. |
| 1632 | |
| 1633 | This gets called by TestSuite as it finds and reads test yaml files. |
| 1634 | Multiple TestCase instances may be generated from a single testcase.yaml, |
| 1635 | each one corresponds to an entry within that file. |
| 1636 | |
| 1637 | We need to have a unique name for every single test case. Since |
| 1638 | a testcase.yaml can define multiple tests, the canonical name for |
| 1639 | the test case is <workdir>/<name>. |
| 1640 | |
| 1641 | @param testcase_root os.path.abspath() of one of the --testcase-root |
| 1642 | @param workdir Sub-directory of testcase_root where the |
| 1643 | .yaml test configuration file was found |
| 1644 | @param name Name of this test case, corresponding to the entry name |
| 1645 | in the test case configuration file. For many test cases that just |
| 1646 | define one test, can be anything and is usually "test". This is |
| 1647 | really only used to distinguish between different cases when |
| 1648 | the testcase.yaml defines multiple tests |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1649 | """ |
| 1650 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1651 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1652 | self.source_dir = "" |
| 1653 | self.yamlfile = "" |
| 1654 | self.cases = [] |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1655 | self.name = self.get_unique(testcase_root, workdir, name) |
| 1656 | self.id = name |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1657 | |
| 1658 | self.type = None |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1659 | self.tags = set() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1660 | self.extra_args = None |
| 1661 | self.extra_configs = None |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 1662 | self.arch_allow = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1663 | self.arch_exclude = None |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1664 | self.skip = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1665 | self.platform_exclude = None |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 1666 | self.platform_allow = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1667 | self.toolchain_exclude = None |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 1668 | self.toolchain_allow = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1669 | self.tc_filter = None |
| 1670 | self.timeout = 60 |
| 1671 | self.harness = "" |
| 1672 | self.harness_config = {} |
| 1673 | self.build_only = True |
| 1674 | self.build_on_all = False |
| 1675 | self.slow = False |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1676 | self.min_ram = -1 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1677 | self.depends_on = None |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1678 | self.min_flash = -1 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1679 | self.extra_sections = None |
Anas Nashif | 1636c31 | 2020-05-28 08:02:54 -0400 | [diff] [blame] | 1680 | self.integration_platforms = [] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1681 | |
| 1682 | @staticmethod |
| 1683 | def get_unique(testcase_root, workdir, name): |
| 1684 | |
| 1685 | canonical_testcase_root = os.path.realpath(testcase_root) |
| 1686 | if Path(canonical_zephyr_base) in Path(canonical_testcase_root).parents: |
| 1687 | # This is in ZEPHYR_BASE, so include path in name for uniqueness |
| 1688 | # FIXME: We should not depend on path of test for unique names. |
| 1689 | relative_tc_root = os.path.relpath(canonical_testcase_root, |
| 1690 | start=canonical_zephyr_base) |
| 1691 | else: |
| 1692 | relative_tc_root = "" |
| 1693 | |
| 1694 | # workdir can be "." |
| 1695 | unique = os.path.normpath(os.path.join(relative_tc_root, workdir, name)) |
Anas Nashif | 7a69125 | 2020-05-07 07:47:51 -0400 | [diff] [blame] | 1696 | check = name.split(".") |
| 1697 | if len(check) < 2: |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 1698 | raise TwisterException(f"""bad test name '{name}' in {testcase_root}/{workdir}. \ |
Anas Nashif | 7a69125 | 2020-05-07 07:47:51 -0400 | [diff] [blame] | 1699 | Tests should reference the category and subsystem with a dot as a separator. |
| 1700 | """ |
| 1701 | ) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1702 | return unique |
| 1703 | |
| 1704 | @staticmethod |
| 1705 | def scan_file(inf_name): |
| 1706 | suite_regex = re.compile( |
| 1707 | # do not match until end-of-line, otherwise we won't allow |
| 1708 | # stc_regex below to catch the ones that are declared in the same |
| 1709 | # line--as we only search starting the end of this match |
| 1710 | br"^\s*ztest_test_suite\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,", |
| 1711 | re.MULTILINE) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1712 | registered_suite_regex = re.compile( |
| 1713 | br"^\s*ztest_register_test_suite" |
| 1714 | br"\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,", |
| 1715 | re.MULTILINE) |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1716 | # Checks if the file contains a definition of "void test_main(void)" |
| 1717 | # Since ztest provides a plain test_main implementation it is OK to: |
| 1718 | # 1. register test suites and not call the run function iff the test |
| 1719 | # doesn't have a custom test_main. |
| 1720 | # 2. register test suites and a custom test_main definition iff the test |
| 1721 | # also calls ztest_run_registered_test_suites. |
| 1722 | test_main_regex = re.compile( |
| 1723 | br"^\s*void\s+test_main\(void\)", |
| 1724 | re.MULTILINE) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1725 | stc_regex = re.compile( |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1726 | br"""^\s* # empy space at the beginning is ok |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1727 | # catch the case where it is declared in the same sentence, e.g: |
| 1728 | # |
| 1729 | # ztest_test_suite(mutex_complex, ztest_user_unit_test(TESTNAME)); |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1730 | # ztest_register_test_suite(n, p, ztest_user_unit_test(TESTNAME), |
| 1731 | (?:ztest_ |
| 1732 | (?:test_suite\(|register_test_suite\([a-zA-Z0-9_]+\s*,\s*) |
| 1733 | [a-zA-Z0-9_]+\s*,\s* |
| 1734 | )? |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1735 | # Catch ztest[_user]_unit_test-[_setup_teardown](TESTNAME) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1736 | ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)? |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1737 | # Consume the argument that becomes the extra testcse |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1738 | \(\s*(?P<stc_name>[a-zA-Z0-9_]+) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1739 | # _setup_teardown() variant has two extra arguments that we ignore |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1740 | (?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)? |
| 1741 | \s*\)""", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1742 | # We don't check how it finishes; we don't care |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1743 | re.MULTILINE | re.VERBOSE) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1744 | suite_run_regex = re.compile( |
| 1745 | br"^\s*ztest_run_test_suite\((?P<suite_name>[a-zA-Z0-9_]+)\)", |
| 1746 | re.MULTILINE) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1747 | registered_suite_run_regex = re.compile( |
| 1748 | br"^\s*ztest_run_registered_test_suites\(" |
| 1749 | br"(\*+|&)?(?P<state_identifier>[a-zA-Z0-9_]+)\)", |
| 1750 | re.MULTILINE) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1751 | achtung_regex = re.compile( |
| 1752 | br"(#ifdef|#endif)", |
| 1753 | re.MULTILINE) |
| 1754 | warnings = None |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1755 | has_registered_test_suites = False |
| 1756 | has_run_registered_test_suites = False |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1757 | has_test_main = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1758 | |
| 1759 | with open(inf_name) as inf: |
| 1760 | if os.name == 'nt': |
| 1761 | mmap_args = {'fileno': inf.fileno(), 'length': 0, 'access': mmap.ACCESS_READ} |
| 1762 | else: |
| 1763 | mmap_args = {'fileno': inf.fileno(), 'length': 0, 'flags': mmap.MAP_PRIVATE, 'prot': mmap.PROT_READ, |
| 1764 | 'offset': 0} |
| 1765 | |
| 1766 | with contextlib.closing(mmap.mmap(**mmap_args)) as main_c: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1767 | suite_regex_match = suite_regex.search(main_c) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1768 | registered_suite_regex_match = registered_suite_regex.search( |
| 1769 | main_c) |
| 1770 | |
| 1771 | if registered_suite_regex_match: |
| 1772 | has_registered_test_suites = True |
| 1773 | if registered_suite_run_regex.search(main_c): |
| 1774 | has_run_registered_test_suites = True |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1775 | if test_main_regex.search(main_c): |
| 1776 | has_test_main = True |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1777 | |
| 1778 | if not suite_regex_match and not has_registered_test_suites: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1779 | # can't find ztest_test_suite, maybe a client, because |
| 1780 | # it includes ztest.h |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1781 | return ScanPathResult( |
| 1782 | matches=None, |
| 1783 | warnings=None, |
| 1784 | has_registered_test_suites=has_registered_test_suites, |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1785 | has_run_registered_test_suites=has_run_registered_test_suites, |
| 1786 | has_test_main=has_test_main) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1787 | |
| 1788 | suite_run_match = suite_run_regex.search(main_c) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1789 | if suite_regex_match and not suite_run_match: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1790 | raise ValueError("can't find ztest_run_test_suite") |
| 1791 | |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1792 | if suite_regex_match: |
| 1793 | search_start = suite_regex_match.end() |
| 1794 | else: |
| 1795 | search_start = registered_suite_regex_match.end() |
| 1796 | |
| 1797 | if suite_run_match: |
| 1798 | search_end = suite_run_match.start() |
| 1799 | else: |
| 1800 | search_end = re.compile(br"\);", re.MULTILINE) \ |
| 1801 | .search(main_c, search_start) \ |
| 1802 | .end() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1803 | achtung_matches = re.findall( |
| 1804 | achtung_regex, |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1805 | main_c[search_start:search_end]) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1806 | if achtung_matches: |
| 1807 | warnings = "found invalid %s in ztest_test_suite()" \ |
Spoorthy Priya Yerabolu | ad4d4fc | 2020-06-25 02:57:05 -0700 | [diff] [blame] | 1808 | % ", ".join(sorted({match.decode() for match in achtung_matches},reverse = True)) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1809 | _matches = re.findall( |
| 1810 | stc_regex, |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1811 | main_c[search_start:search_end]) |
Anas Nashif | 44f7ba0 | 2020-05-12 12:26:41 -0400 | [diff] [blame] | 1812 | for match in _matches: |
| 1813 | if not match.decode().startswith("test_"): |
| 1814 | warnings = "Found a test that does not start with test_" |
Maciej Perkowski | 034d4f2 | 2020-08-03 14:11:11 +0200 | [diff] [blame] | 1815 | matches = [match.decode().replace("test_", "", 1) for match in _matches] |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1816 | return ScanPathResult( |
| 1817 | matches=matches, |
| 1818 | warnings=warnings, |
| 1819 | has_registered_test_suites=has_registered_test_suites, |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1820 | has_run_registered_test_suites=has_run_registered_test_suites, |
| 1821 | has_test_main=has_test_main) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1822 | |
| 1823 | def scan_path(self, path): |
| 1824 | subcases = [] |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1825 | has_registered_test_suites = False |
| 1826 | has_run_registered_test_suites = False |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1827 | has_test_main = False |
Anas Nashif | 91fd68d | 2020-05-08 07:22:58 -0400 | [diff] [blame] | 1828 | for filename in glob.glob(os.path.join(path, "src", "*.c*")): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1829 | try: |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1830 | result: ScanPathResult = self.scan_file(filename) |
| 1831 | if result.warnings: |
| 1832 | logger.error("%s: %s" % (filename, result.warnings)) |
| 1833 | raise TwisterRuntimeError( |
| 1834 | "%s: %s" % (filename, result.warnings)) |
| 1835 | if result.matches: |
| 1836 | subcases += result.matches |
| 1837 | if result.has_registered_test_suites: |
| 1838 | has_registered_test_suites = True |
| 1839 | if result.has_run_registered_test_suites: |
| 1840 | has_run_registered_test_suites = True |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1841 | if result.has_test_main: |
| 1842 | has_test_main = True |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1843 | except ValueError as e: |
| 1844 | logger.error("%s: can't find: %s" % (filename, e)) |
Anas Nashif | 61c6e2b | 2020-05-07 07:03:30 -0400 | [diff] [blame] | 1845 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1846 | for filename in glob.glob(os.path.join(path, "*.c")): |
| 1847 | try: |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1848 | result: ScanPathResult = self.scan_file(filename) |
| 1849 | if result.warnings: |
| 1850 | logger.error("%s: %s" % (filename, result.warnings)) |
| 1851 | if result.matches: |
| 1852 | subcases += result.matches |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1853 | except ValueError as e: |
| 1854 | logger.error("%s: can't find: %s" % (filename, e)) |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1855 | |
Yuval Peress | 27f6a5e | 2021-08-01 22:12:44 -0600 | [diff] [blame] | 1856 | if (has_registered_test_suites and has_test_main and |
| 1857 | not has_run_registered_test_suites): |
Yuval Peress | dee79d2 | 2021-07-23 11:47:52 -0600 | [diff] [blame] | 1858 | warning = \ |
| 1859 | "Found call to 'ztest_register_test_suite()' but no "\ |
| 1860 | "call to 'ztest_run_registered_test_suites()'" |
| 1861 | logger.error(warning) |
| 1862 | raise TwisterRuntimeError(warning) |
| 1863 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1864 | return subcases |
| 1865 | |
| 1866 | def parse_subcases(self, test_path): |
| 1867 | results = self.scan_path(test_path) |
| 1868 | for sub in results: |
| 1869 | name = "{}.{}".format(self.id, sub) |
| 1870 | self.cases.append(name) |
| 1871 | |
| 1872 | if not results: |
| 1873 | self.cases.append(self.id) |
| 1874 | |
| 1875 | def __str__(self): |
| 1876 | return self.name |
| 1877 | |
| 1878 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1879 | class TestInstance(DisablePyTestCollectionMixin): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1880 | """Class representing the execution of a particular TestCase on a platform |
| 1881 | |
| 1882 | @param test The TestCase object we want to build/execute |
| 1883 | @param platform Platform object that we want to build and run against |
| 1884 | @param base_outdir Base directory for all test results. The actual |
| 1885 | out directory used is <outdir>/<platform>/<test case name> |
| 1886 | """ |
| 1887 | |
| 1888 | def __init__(self, testcase, platform, outdir): |
| 1889 | |
| 1890 | self.testcase = testcase |
| 1891 | self.platform = platform |
| 1892 | |
| 1893 | self.status = None |
| 1894 | self.reason = "Unknown" |
| 1895 | self.metrics = dict() |
| 1896 | self.handler = None |
| 1897 | self.outdir = outdir |
| 1898 | |
| 1899 | self.name = os.path.join(platform.name, testcase.name) |
| 1900 | self.build_dir = os.path.join(outdir, platform.name, testcase.name) |
| 1901 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1902 | self.run = False |
| 1903 | |
| 1904 | self.results = {} |
| 1905 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 1906 | def __getstate__(self): |
| 1907 | d = self.__dict__.copy() |
| 1908 | return d |
| 1909 | |
| 1910 | def __setstate__(self, d): |
| 1911 | self.__dict__.update(d) |
| 1912 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1913 | def __lt__(self, other): |
| 1914 | return self.name < other.name |
| 1915 | |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1916 | |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1917 | @staticmethod |
| 1918 | def testcase_runnable(testcase, fixtures): |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1919 | can_run = False |
| 1920 | # console harness allows us to run the test and capture data. |
YouhuaX Zhu | 965c8b9 | 2020-10-22 09:37:27 +0800 | [diff] [blame] | 1921 | if testcase.harness in [ 'console', 'ztest', 'pytest']: |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1922 | can_run = True |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1923 | # if we have a fixture that is also being supplied on the |
| 1924 | # command-line, then we need to run the test, not just build it. |
| 1925 | fixture = testcase.harness_config.get('fixture') |
| 1926 | if fixture: |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1927 | can_run = (fixture in fixtures) |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1928 | |
| 1929 | elif testcase.harness: |
| 1930 | can_run = False |
| 1931 | else: |
| 1932 | can_run = True |
| 1933 | |
| 1934 | return can_run |
| 1935 | |
| 1936 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 1937 | # Global testsuite parameters |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1938 | def check_runnable(self, enable_slow=False, filter='buildable', fixtures=[]): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1939 | |
| 1940 | # right now we only support building on windows. running is still work |
| 1941 | # in progress. |
| 1942 | if os.name == 'nt': |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1943 | return False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1944 | |
| 1945 | # we asked for build-only on the command line |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1946 | if self.testcase.build_only: |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1947 | return False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1948 | |
| 1949 | # Do not run slow tests: |
| 1950 | skip_slow = self.testcase.slow and not enable_slow |
| 1951 | if skip_slow: |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1952 | return False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1953 | |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1954 | target_ready = bool(self.testcase.type == "unit" or \ |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1955 | self.platform.type == "native" or \ |
Jaxson Han | 8af11d4 | 2021-02-24 10:23:46 +0800 | [diff] [blame] | 1956 | self.platform.simulation in ["mdb-nsim", "nsim", "renode", "qemu", "tsim", "armfvp"] or \ |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1957 | filter == 'runnable') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1958 | |
| 1959 | if self.platform.simulation == "nsim": |
| 1960 | if not find_executable("nsimdrv"): |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1961 | target_ready = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1962 | |
Eugeniy Paltsev | 91d7ec5 | 2020-10-07 22:12:42 +0300 | [diff] [blame] | 1963 | if self.platform.simulation == "mdb-nsim": |
Anas Nashif | 0f831b1 | 2020-09-03 12:33:16 -0400 | [diff] [blame] | 1964 | if not find_executable("mdb"): |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 1965 | target_ready = False |
Anas Nashif | 0f831b1 | 2020-09-03 12:33:16 -0400 | [diff] [blame] | 1966 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1967 | if self.platform.simulation == "renode": |
| 1968 | if not find_executable("renode"): |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1969 | target_ready = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1970 | |
Martin Åberg | c107714 | 2020-10-19 18:21:52 +0200 | [diff] [blame] | 1971 | if self.platform.simulation == "tsim": |
| 1972 | if not find_executable("tsim-leon3"): |
| 1973 | target_ready = False |
| 1974 | |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1975 | testcase_runnable = self.testcase_runnable(self.testcase, fixtures) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1976 | |
Anas Nashif | 4ca0b95 | 2020-07-24 09:22:25 -0400 | [diff] [blame] | 1977 | return testcase_runnable and target_ready |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1978 | |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 1979 | def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable_coverage=False, coverage_platform=[]): |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 1980 | # Create this in a "twister/" subdirectory otherwise this |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1981 | # will pass this overlay to kconfig.py *twice* and kconfig.cmake |
| 1982 | # will silently give that second time precedence over any |
| 1983 | # --extra-args=CONFIG_* |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 1984 | subdir = os.path.join(self.build_dir, "twister") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1985 | |
Kumar Gala | 51d6931 | 2020-09-24 13:28:50 -0500 | [diff] [blame] | 1986 | content = "" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1987 | |
Kumar Gala | 51d6931 | 2020-09-24 13:28:50 -0500 | [diff] [blame] | 1988 | if self.testcase.extra_configs: |
| 1989 | content = "\n".join(self.testcase.extra_configs) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1990 | |
Kumar Gala | 51d6931 | 2020-09-24 13:28:50 -0500 | [diff] [blame] | 1991 | if enable_coverage: |
| 1992 | if platform.name in coverage_platform: |
| 1993 | content = content + "\nCONFIG_COVERAGE=y" |
| 1994 | content = content + "\nCONFIG_COVERAGE_DUMP=y" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1995 | |
Kumar Gala | 51d6931 | 2020-09-24 13:28:50 -0500 | [diff] [blame] | 1996 | if enable_asan: |
| 1997 | if platform.type == "native": |
| 1998 | content = content + "\nCONFIG_ASAN=y" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 1999 | |
Kumar Gala | 51d6931 | 2020-09-24 13:28:50 -0500 | [diff] [blame] | 2000 | if enable_ubsan: |
| 2001 | if platform.type == "native": |
| 2002 | content = content + "\nCONFIG_UBSAN=y" |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 2003 | |
Kumar Gala | 51d6931 | 2020-09-24 13:28:50 -0500 | [diff] [blame] | 2004 | if content: |
| 2005 | os.makedirs(subdir, exist_ok=True) |
| 2006 | file = os.path.join(subdir, "testcase_extra.conf") |
| 2007 | with open(file, "w") as f: |
| 2008 | f.write(content) |
| 2009 | |
| 2010 | return content |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2011 | |
| 2012 | def calculate_sizes(self): |
| 2013 | """Get the RAM/ROM sizes of a test case. |
| 2014 | |
| 2015 | This can only be run after the instance has been executed by |
| 2016 | MakeGenerator, otherwise there won't be any binaries to measure. |
| 2017 | |
| 2018 | @return A SizeCalculator object |
| 2019 | """ |
| 2020 | fns = glob.glob(os.path.join(self.build_dir, "zephyr", "*.elf")) |
| 2021 | fns.extend(glob.glob(os.path.join(self.build_dir, "zephyr", "*.exe"))) |
| 2022 | fns = [x for x in fns if not x.endswith('_prebuilt.elf')] |
| 2023 | if len(fns) != 1: |
| 2024 | raise BuildError("Missing/multiple output ELF binary") |
| 2025 | |
| 2026 | return SizeCalculator(fns[0], self.testcase.extra_sections) |
| 2027 | |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 2028 | def fill_results_by_status(self): |
| 2029 | """Fills results according to self.status |
| 2030 | |
| 2031 | The method is used to propagate the instance level status |
| 2032 | to the test cases inside. Useful when the whole instance is skipped |
| 2033 | and the info is required also at the test cases level for reporting. |
| 2034 | Should be used with caution, e.g. should not be used |
| 2035 | to fill all results with passes |
| 2036 | """ |
| 2037 | status_to_verdict = { |
| 2038 | 'skipped': 'SKIP', |
| 2039 | 'error': 'BLOCK', |
| 2040 | 'failure': 'FAILED' |
| 2041 | } |
| 2042 | |
| 2043 | for k in self.results: |
| 2044 | self.results[k] = status_to_verdict[self.status] |
| 2045 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2046 | def __repr__(self): |
| 2047 | return "<TestCase %s on %s>" % (self.testcase.name, self.platform.name) |
| 2048 | |
| 2049 | |
| 2050 | class CMake(): |
| 2051 | config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') |
| 2052 | dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') |
| 2053 | |
| 2054 | def __init__(self, testcase, platform, source_dir, build_dir): |
| 2055 | |
| 2056 | self.cwd = None |
| 2057 | self.capture_output = True |
| 2058 | |
| 2059 | self.defconfig = {} |
| 2060 | self.cmake_cache = {} |
| 2061 | |
| 2062 | self.instance = None |
| 2063 | self.testcase = testcase |
| 2064 | self.platform = platform |
| 2065 | self.source_dir = source_dir |
| 2066 | self.build_dir = build_dir |
| 2067 | self.log = "build.log" |
| 2068 | self.generator = None |
| 2069 | self.generator_cmd = None |
| 2070 | |
| 2071 | def parse_generated(self): |
| 2072 | self.defconfig = {} |
| 2073 | return {} |
| 2074 | |
| 2075 | def run_build(self, args=[]): |
| 2076 | |
| 2077 | logger.debug("Building %s for %s" % (self.source_dir, self.platform.name)) |
| 2078 | |
| 2079 | cmake_args = [] |
| 2080 | cmake_args.extend(args) |
| 2081 | cmake = shutil.which('cmake') |
| 2082 | cmd = [cmake] + cmake_args |
| 2083 | kwargs = dict() |
| 2084 | |
| 2085 | if self.capture_output: |
| 2086 | kwargs['stdout'] = subprocess.PIPE |
| 2087 | # CMake sends the output of message() to stderr unless it's STATUS |
| 2088 | kwargs['stderr'] = subprocess.STDOUT |
| 2089 | |
| 2090 | if self.cwd: |
| 2091 | kwargs['cwd'] = self.cwd |
| 2092 | |
| 2093 | p = subprocess.Popen(cmd, **kwargs) |
| 2094 | out, _ = p.communicate() |
| 2095 | |
| 2096 | results = {} |
| 2097 | if p.returncode == 0: |
| 2098 | msg = "Finished building %s for %s" % (self.source_dir, self.platform.name) |
| 2099 | |
| 2100 | self.instance.status = "passed" |
| 2101 | results = {'msg': msg, "returncode": p.returncode, "instance": self.instance} |
| 2102 | |
| 2103 | if out: |
| 2104 | log_msg = out.decode(sys.getdefaultencoding()) |
| 2105 | with open(os.path.join(self.build_dir, self.log), "a") as log: |
| 2106 | log.write(log_msg) |
| 2107 | |
| 2108 | else: |
| 2109 | return None |
| 2110 | else: |
| 2111 | # A real error occurred, raise an exception |
Anas Nashif | 5a6e64f | 2020-10-30 13:01:50 -0400 | [diff] [blame] | 2112 | log_msg = "" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2113 | if out: |
| 2114 | log_msg = out.decode(sys.getdefaultencoding()) |
| 2115 | with open(os.path.join(self.build_dir, self.log), "a") as log: |
| 2116 | log.write(log_msg) |
| 2117 | |
| 2118 | if log_msg: |
Henrik Brix Andersen | 9aa4a70 | 2021-09-27 21:21:03 +0200 | [diff] [blame] | 2119 | res = re.findall("region `(FLASH|ROM|RAM|ICCM|DCCM|SRAM)' overflowed by", log_msg) |
Anas Nashif | f68146f | 2020-11-28 11:07:06 -0500 | [diff] [blame] | 2120 | if res and not self.overflow_as_errors: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2121 | logger.debug("Test skipped due to {} Overflow".format(res[0])) |
| 2122 | self.instance.status = "skipped" |
| 2123 | self.instance.reason = "{} overflow".format(res[0]) |
| 2124 | else: |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 2125 | self.instance.status = "error" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2126 | self.instance.reason = "Build failure" |
| 2127 | |
| 2128 | results = { |
| 2129 | "returncode": p.returncode, |
| 2130 | "instance": self.instance, |
| 2131 | } |
| 2132 | |
| 2133 | return results |
| 2134 | |
| 2135 | def run_cmake(self, args=[]): |
| 2136 | |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2137 | if self.warnings_as_errors: |
| 2138 | ldflags = "-Wl,--fatal-warnings" |
| 2139 | cflags = "-Werror" |
| 2140 | aflags = "-Wa,--fatal-warnings" |
Martí Bolívar | c4079e4 | 2021-07-30 15:43:27 -0700 | [diff] [blame] | 2141 | gen_defines_args = "--edtlib-Werror" |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2142 | else: |
| 2143 | ldflags = cflags = aflags = "" |
Martí Bolívar | fbd34dc | 2021-02-14 18:02:12 -0800 | [diff] [blame] | 2144 | gen_defines_args = "" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2145 | |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2146 | logger.debug("Running cmake on %s for %s" % (self.source_dir, self.platform.name)) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2147 | cmake_args = [ |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2148 | f'-B{self.build_dir}', |
| 2149 | f'-S{self.source_dir}', |
| 2150 | f'-DEXTRA_CFLAGS="{cflags}"', |
| 2151 | f'-DEXTRA_AFLAGS="{aflags}', |
| 2152 | f'-DEXTRA_LDFLAGS="{ldflags}"', |
Martí Bolívar | fbd34dc | 2021-02-14 18:02:12 -0800 | [diff] [blame] | 2153 | f'-DEXTRA_GEN_DEFINES_ARGS={gen_defines_args}', |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2154 | f'-G{self.generator}' |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2155 | ] |
| 2156 | |
| 2157 | if self.cmake_only: |
| 2158 | cmake_args.append("-DCMAKE_EXPORT_COMPILE_COMMANDS=1") |
| 2159 | |
| 2160 | args = ["-D{}".format(a.replace('"', '')) for a in args] |
| 2161 | cmake_args.extend(args) |
| 2162 | |
| 2163 | cmake_opts = ['-DBOARD={}'.format(self.platform.name)] |
| 2164 | cmake_args.extend(cmake_opts) |
| 2165 | |
| 2166 | |
| 2167 | logger.debug("Calling cmake with arguments: {}".format(cmake_args)) |
| 2168 | cmake = shutil.which('cmake') |
| 2169 | cmd = [cmake] + cmake_args |
| 2170 | kwargs = dict() |
| 2171 | |
| 2172 | if self.capture_output: |
| 2173 | kwargs['stdout'] = subprocess.PIPE |
| 2174 | # CMake sends the output of message() to stderr unless it's STATUS |
| 2175 | kwargs['stderr'] = subprocess.STDOUT |
| 2176 | |
| 2177 | if self.cwd: |
| 2178 | kwargs['cwd'] = self.cwd |
| 2179 | |
| 2180 | p = subprocess.Popen(cmd, **kwargs) |
| 2181 | out, _ = p.communicate() |
| 2182 | |
| 2183 | if p.returncode == 0: |
| 2184 | filter_results = self.parse_generated() |
| 2185 | msg = "Finished building %s for %s" % (self.source_dir, self.platform.name) |
| 2186 | logger.debug(msg) |
| 2187 | results = {'msg': msg, 'filter': filter_results} |
| 2188 | |
| 2189 | else: |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 2190 | self.instance.status = "error" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2191 | self.instance.reason = "Cmake build failure" |
Piotr Golyzniak | 35a0520 | 2021-07-23 11:17:11 +0200 | [diff] [blame] | 2192 | self.instance.fill_results_by_status() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2193 | logger.error("Cmake build failure: %s for %s" % (self.source_dir, self.platform.name)) |
| 2194 | results = {"returncode": p.returncode} |
| 2195 | |
| 2196 | if out: |
| 2197 | with open(os.path.join(self.build_dir, self.log), "a") as log: |
| 2198 | log_msg = out.decode(sys.getdefaultencoding()) |
| 2199 | log.write(log_msg) |
| 2200 | |
| 2201 | return results |
| 2202 | |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 2203 | @staticmethod |
| 2204 | def run_cmake_script(args=[]): |
| 2205 | |
| 2206 | logger.debug("Running cmake script %s" % (args[0])) |
| 2207 | |
| 2208 | cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]] |
| 2209 | cmake_args.extend(['-P', args[0]]) |
| 2210 | |
| 2211 | logger.debug("Calling cmake with arguments: {}".format(cmake_args)) |
| 2212 | cmake = shutil.which('cmake') |
Steven Huang | 1413c55 | 2021-03-14 01:54:40 -0500 | [diff] [blame] | 2213 | if not cmake: |
| 2214 | msg = "Unable to find `cmake` in path" |
| 2215 | logger.error(msg) |
| 2216 | raise Exception(msg) |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 2217 | cmd = [cmake] + cmake_args |
| 2218 | |
| 2219 | kwargs = dict() |
| 2220 | kwargs['stdout'] = subprocess.PIPE |
| 2221 | # CMake sends the output of message() to stderr unless it's STATUS |
| 2222 | kwargs['stderr'] = subprocess.STDOUT |
| 2223 | |
| 2224 | p = subprocess.Popen(cmd, **kwargs) |
| 2225 | out, _ = p.communicate() |
| 2226 | |
Benedikt Schmidt | b83681d | 2021-08-23 14:35:55 +0200 | [diff] [blame] | 2227 | # It might happen that the environment adds ANSI escape codes like \x1b[0m, |
| 2228 | # for instance if twister is executed from inside a makefile. In such a |
| 2229 | # scenario it is then necessary to remove them, as otherwise the JSON decoding |
| 2230 | # will fail. |
| 2231 | ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') |
| 2232 | out = ansi_escape.sub('', out.decode()) |
| 2233 | |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 2234 | if p.returncode == 0: |
| 2235 | msg = "Finished running %s" % (args[0]) |
| 2236 | logger.debug(msg) |
| 2237 | results = {"returncode": p.returncode, "msg": msg, "stdout": out} |
| 2238 | |
| 2239 | else: |
| 2240 | logger.error("Cmake script failure: %s" % (args[0])) |
Torsten Rasmussen | af87599 | 2021-09-28 13:04:52 +0200 | [diff] [blame] | 2241 | results = {"returncode": p.returncode, "returnmsg": out} |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 2242 | |
| 2243 | return results |
| 2244 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2245 | |
| 2246 | class FilterBuilder(CMake): |
| 2247 | |
| 2248 | def __init__(self, testcase, platform, source_dir, build_dir): |
| 2249 | super().__init__(testcase, platform, source_dir, build_dir) |
| 2250 | |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 2251 | self.log = "config-twister.log" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2252 | |
| 2253 | def parse_generated(self): |
| 2254 | |
| 2255 | if self.platform.name == "unit_testing": |
| 2256 | return {} |
| 2257 | |
| 2258 | cmake_cache_path = os.path.join(self.build_dir, "CMakeCache.txt") |
| 2259 | defconfig_path = os.path.join(self.build_dir, "zephyr", ".config") |
| 2260 | |
| 2261 | with open(defconfig_path, "r") as fp: |
| 2262 | defconfig = {} |
| 2263 | for line in fp.readlines(): |
| 2264 | m = self.config_re.match(line) |
| 2265 | if not m: |
| 2266 | if line.strip() and not line.startswith("#"): |
| 2267 | sys.stderr.write("Unrecognized line %s\n" % line) |
| 2268 | continue |
| 2269 | defconfig[m.group(1)] = m.group(2).strip() |
| 2270 | |
| 2271 | self.defconfig = defconfig |
| 2272 | |
| 2273 | cmake_conf = {} |
| 2274 | try: |
| 2275 | cache = CMakeCache.from_file(cmake_cache_path) |
| 2276 | except FileNotFoundError: |
| 2277 | cache = {} |
| 2278 | |
| 2279 | for k in iter(cache): |
| 2280 | cmake_conf[k.name] = k.value |
| 2281 | |
| 2282 | self.cmake_cache = cmake_conf |
| 2283 | |
| 2284 | filter_data = { |
| 2285 | "ARCH": self.platform.arch, |
| 2286 | "PLATFORM": self.platform.name |
| 2287 | } |
| 2288 | filter_data.update(os.environ) |
| 2289 | filter_data.update(self.defconfig) |
| 2290 | filter_data.update(self.cmake_cache) |
| 2291 | |
Martí Bolívar | 9c92baa | 2020-07-08 14:43:07 -0700 | [diff] [blame] | 2292 | edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2293 | if self.testcase and self.testcase.tc_filter: |
| 2294 | try: |
Martí Bolívar | 9c92baa | 2020-07-08 14:43:07 -0700 | [diff] [blame] | 2295 | if os.path.exists(edt_pickle): |
| 2296 | with open(edt_pickle, 'rb') as f: |
| 2297 | edt = pickle.load(f) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2298 | else: |
| 2299 | edt = None |
| 2300 | res = expr_parser.parse(self.testcase.tc_filter, filter_data, edt) |
| 2301 | |
| 2302 | except (ValueError, SyntaxError) as se: |
| 2303 | sys.stderr.write( |
| 2304 | "Failed processing %s\n" % self.testcase.yamlfile) |
| 2305 | raise se |
| 2306 | |
| 2307 | if not res: |
| 2308 | return {os.path.join(self.platform.name, self.testcase.name): True} |
| 2309 | else: |
| 2310 | return {os.path.join(self.platform.name, self.testcase.name): False} |
| 2311 | else: |
| 2312 | self.platform.filter_data = filter_data |
| 2313 | return filter_data |
| 2314 | |
| 2315 | |
| 2316 | class ProjectBuilder(FilterBuilder): |
| 2317 | |
| 2318 | def __init__(self, suite, instance, **kwargs): |
| 2319 | super().__init__(instance.testcase, instance.platform, instance.testcase.source_dir, instance.build_dir) |
| 2320 | |
| 2321 | self.log = "build.log" |
| 2322 | self.instance = instance |
| 2323 | self.suite = suite |
Maciej Perkowski | c67a0cd | 2020-08-13 15:20:13 +0200 | [diff] [blame] | 2324 | self.filtered_tests = 0 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2325 | |
| 2326 | self.lsan = kwargs.get('lsan', False) |
| 2327 | self.asan = kwargs.get('asan', False) |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 2328 | self.ubsan = kwargs.get('ubsan', False) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2329 | self.valgrind = kwargs.get('valgrind', False) |
| 2330 | self.extra_args = kwargs.get('extra_args', []) |
| 2331 | self.device_testing = kwargs.get('device_testing', False) |
| 2332 | self.cmake_only = kwargs.get('cmake_only', False) |
| 2333 | self.cleanup = kwargs.get('cleanup', False) |
| 2334 | self.coverage = kwargs.get('coverage', False) |
| 2335 | self.inline_logs = kwargs.get('inline_logs', False) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2336 | self.generator = kwargs.get('generator', None) |
| 2337 | self.generator_cmd = kwargs.get('generator_cmd', None) |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 2338 | self.verbose = kwargs.get('verbose', None) |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2339 | self.warnings_as_errors = kwargs.get('warnings_as_errors', True) |
Anas Nashif | f68146f | 2020-11-28 11:07:06 -0500 | [diff] [blame] | 2340 | self.overflow_as_errors = kwargs.get('overflow_as_errors', False) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2341 | |
| 2342 | @staticmethod |
| 2343 | def log_info(filename, inline_logs): |
| 2344 | filename = os.path.abspath(os.path.realpath(filename)) |
| 2345 | if inline_logs: |
| 2346 | logger.info("{:-^100}".format(filename)) |
| 2347 | |
| 2348 | try: |
| 2349 | with open(filename) as fp: |
| 2350 | data = fp.read() |
| 2351 | except Exception as e: |
| 2352 | data = "Unable to read log data (%s)\n" % (str(e)) |
| 2353 | |
| 2354 | logger.error(data) |
| 2355 | |
| 2356 | logger.info("{:-^100}".format(filename)) |
| 2357 | else: |
| 2358 | logger.error("see: " + Fore.YELLOW + filename + Fore.RESET) |
| 2359 | |
| 2360 | def log_info_file(self, inline_logs): |
| 2361 | build_dir = self.instance.build_dir |
| 2362 | h_log = "{}/handler.log".format(build_dir) |
| 2363 | b_log = "{}/build.log".format(build_dir) |
| 2364 | v_log = "{}/valgrind.log".format(build_dir) |
| 2365 | d_log = "{}/device.log".format(build_dir) |
| 2366 | |
| 2367 | if os.path.exists(v_log) and "Valgrind" in self.instance.reason: |
| 2368 | self.log_info("{}".format(v_log), inline_logs) |
| 2369 | elif os.path.exists(h_log) and os.path.getsize(h_log) > 0: |
| 2370 | self.log_info("{}".format(h_log), inline_logs) |
| 2371 | elif os.path.exists(d_log) and os.path.getsize(d_log) > 0: |
| 2372 | self.log_info("{}".format(d_log), inline_logs) |
| 2373 | else: |
| 2374 | self.log_info("{}".format(b_log), inline_logs) |
| 2375 | |
| 2376 | def setup_handler(self): |
| 2377 | |
| 2378 | instance = self.instance |
| 2379 | args = [] |
| 2380 | |
| 2381 | # FIXME: Needs simplification |
| 2382 | if instance.platform.simulation == "qemu": |
| 2383 | instance.handler = QEMUHandler(instance, "qemu") |
| 2384 | args.append("QEMU_PIPE=%s" % instance.handler.get_fifo()) |
| 2385 | instance.handler.call_make_run = True |
| 2386 | elif instance.testcase.type == "unit": |
| 2387 | instance.handler = BinaryHandler(instance, "unit") |
| 2388 | instance.handler.binary = os.path.join(instance.build_dir, "testbinary") |
Anas Nashif | 051602f | 2020-04-28 14:27:46 -0400 | [diff] [blame] | 2389 | if self.coverage: |
| 2390 | args.append("COVERAGE=1") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2391 | elif instance.platform.type == "native": |
| 2392 | handler = BinaryHandler(instance, "native") |
| 2393 | |
| 2394 | handler.asan = self.asan |
| 2395 | handler.valgrind = self.valgrind |
| 2396 | handler.lsan = self.lsan |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 2397 | handler.ubsan = self.ubsan |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2398 | handler.coverage = self.coverage |
| 2399 | |
| 2400 | handler.binary = os.path.join(instance.build_dir, "zephyr", "zephyr.exe") |
| 2401 | instance.handler = handler |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2402 | elif instance.platform.simulation == "renode": |
| 2403 | if find_executable("renode"): |
| 2404 | instance.handler = BinaryHandler(instance, "renode") |
| 2405 | instance.handler.pid_fn = os.path.join(instance.build_dir, "renode.pid") |
| 2406 | instance.handler.call_make_run = True |
Martin Åberg | c107714 | 2020-10-19 18:21:52 +0200 | [diff] [blame] | 2407 | elif instance.platform.simulation == "tsim": |
| 2408 | instance.handler = BinaryHandler(instance, "tsim") |
| 2409 | instance.handler.call_make_run = True |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2410 | elif self.device_testing: |
| 2411 | instance.handler = DeviceHandler(instance, "device") |
Andrei Emeltchenko | 10fa48d | 2021-01-12 09:56:32 +0200 | [diff] [blame] | 2412 | instance.handler.coverage = self.coverage |
Eugeniy Paltsev | 9f2006c | 2020-10-08 22:31:19 +0300 | [diff] [blame] | 2413 | elif instance.platform.simulation == "nsim": |
| 2414 | if find_executable("nsimdrv"): |
| 2415 | instance.handler = BinaryHandler(instance, "nsim") |
| 2416 | instance.handler.call_make_run = True |
Eugeniy Paltsev | 16032b67 | 2020-10-08 22:42:02 +0300 | [diff] [blame] | 2417 | elif instance.platform.simulation == "mdb-nsim": |
| 2418 | if find_executable("mdb"): |
| 2419 | instance.handler = BinaryHandler(instance, "nsim") |
Jingru Wang | b78d674 | 2021-09-01 09:53:57 +0800 | [diff] [blame] | 2420 | instance.handler.call_make_run = True |
Jaxson Han | 8af11d4 | 2021-02-24 10:23:46 +0800 | [diff] [blame] | 2421 | elif instance.platform.simulation == "armfvp": |
| 2422 | instance.handler = BinaryHandler(instance, "armfvp") |
| 2423 | instance.handler.call_make_run = True |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2424 | |
| 2425 | if instance.handler: |
| 2426 | instance.handler.args = args |
Anas Nashif | b366949 | 2020-03-24 22:33:50 -0400 | [diff] [blame] | 2427 | instance.handler.generator_cmd = self.generator_cmd |
| 2428 | instance.handler.generator = self.generator |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2429 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2430 | def process(self, pipeline, done, message, lock, results): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2431 | op = message.get('op') |
| 2432 | |
| 2433 | if not self.instance.handler: |
| 2434 | self.setup_handler() |
| 2435 | |
| 2436 | # The build process, call cmake and build with configured generator |
| 2437 | if op == "cmake": |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2438 | res = self.cmake() |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 2439 | if self.instance.status in ["failed", "error"]: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2440 | pipeline.put({"op": "report", "test": self.instance}) |
| 2441 | elif self.cmake_only: |
Kumar Gala | 659b24b | 2020-10-09 09:51:02 -0500 | [diff] [blame] | 2442 | if self.instance.status is None: |
| 2443 | self.instance.status = "passed" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2444 | pipeline.put({"op": "report", "test": self.instance}) |
| 2445 | else: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2446 | if self.instance.name in res['filter'] and res['filter'][self.instance.name]: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2447 | logger.debug("filtering %s" % self.instance.name) |
| 2448 | self.instance.status = "skipped" |
| 2449 | self.instance.reason = "filter" |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2450 | results.skipped_runtime += 1 |
Maciej Perkowski | b2fa99c | 2020-05-21 14:45:29 +0200 | [diff] [blame] | 2451 | for case in self.instance.testcase.cases: |
| 2452 | self.instance.results.update({case: 'SKIP'}) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2453 | pipeline.put({"op": "report", "test": self.instance}) |
| 2454 | else: |
| 2455 | pipeline.put({"op": "build", "test": self.instance}) |
| 2456 | |
| 2457 | elif op == "build": |
| 2458 | logger.debug("build test: %s" % self.instance.name) |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2459 | res = self.build() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2460 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2461 | if not res: |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 2462 | self.instance.status = "error" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2463 | self.instance.reason = "Build Failure" |
| 2464 | pipeline.put({"op": "report", "test": self.instance}) |
| 2465 | else: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2466 | # Count skipped cases during build, for example |
| 2467 | # due to ram/rom overflow. |
| 2468 | inst = res.get("instance", None) |
| 2469 | if inst and inst.status == "skipped": |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2470 | results.skipped_runtime += 1 |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2471 | |
| 2472 | if res.get('returncode', 1) > 0: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2473 | pipeline.put({"op": "report", "test": self.instance}) |
| 2474 | else: |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 2475 | if self.instance.run and self.instance.handler: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2476 | pipeline.put({"op": "run", "test": self.instance}) |
| 2477 | else: |
| 2478 | pipeline.put({"op": "report", "test": self.instance}) |
| 2479 | # Run the generated binary using one of the supported handlers |
| 2480 | elif op == "run": |
| 2481 | logger.debug("run test: %s" % self.instance.name) |
| 2482 | self.run() |
| 2483 | self.instance.status, _ = self.instance.handler.get_state() |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2484 | logger.debug(f"run status: {self.instance.name} {self.instance.status}") |
| 2485 | |
| 2486 | # to make it work with pickle |
| 2487 | self.instance.handler.thread = None |
| 2488 | self.instance.handler.suite = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2489 | pipeline.put({ |
| 2490 | "op": "report", |
| 2491 | "test": self.instance, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2492 | "status": self.instance.status, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2493 | "reason": self.instance.reason |
| 2494 | } |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2495 | ) |
| 2496 | |
| 2497 | # Report results and output progress to screen |
| 2498 | elif op == "report": |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2499 | with lock: |
| 2500 | done.put(self.instance) |
| 2501 | self.report_out(results) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2502 | |
| 2503 | if self.cleanup and not self.coverage and self.instance.status == "passed": |
| 2504 | pipeline.put({ |
| 2505 | "op": "cleanup", |
| 2506 | "test": self.instance |
| 2507 | }) |
| 2508 | |
| 2509 | elif op == "cleanup": |
Kumar Gala | 1285c1f | 2020-09-24 13:21:07 -0500 | [diff] [blame] | 2510 | if self.device_testing: |
| 2511 | self.cleanup_device_testing_artifacts() |
| 2512 | else: |
| 2513 | self.cleanup_artifacts() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2514 | |
Kumar Gala | eb2e89a | 2020-10-20 15:44:15 -0500 | [diff] [blame] | 2515 | def cleanup_artifacts(self, additional_keep=[]): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2516 | logger.debug("Cleaning up {}".format(self.instance.build_dir)) |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 2517 | allow = [ |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2518 | 'zephyr/.config', |
| 2519 | 'handler.log', |
| 2520 | 'build.log', |
| 2521 | 'device.log', |
Anas Nashif | 9ace63e | 2020-04-28 07:14:43 -0400 | [diff] [blame] | 2522 | 'recording.csv', |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2523 | ] |
Kumar Gala | 1285c1f | 2020-09-24 13:21:07 -0500 | [diff] [blame] | 2524 | |
| 2525 | allow += additional_keep |
| 2526 | |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 2527 | allow = [os.path.join(self.instance.build_dir, file) for file in allow] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2528 | |
| 2529 | for dirpath, dirnames, filenames in os.walk(self.instance.build_dir, topdown=False): |
| 2530 | for name in filenames: |
| 2531 | path = os.path.join(dirpath, name) |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 2532 | if path not in allow: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2533 | os.remove(path) |
| 2534 | # Remove empty directories and symbolic links to directories |
| 2535 | for dir in dirnames: |
| 2536 | path = os.path.join(dirpath, dir) |
| 2537 | if os.path.islink(path): |
| 2538 | os.remove(path) |
| 2539 | elif not os.listdir(path): |
| 2540 | os.rmdir(path) |
| 2541 | |
Kumar Gala | 1285c1f | 2020-09-24 13:21:07 -0500 | [diff] [blame] | 2542 | def cleanup_device_testing_artifacts(self): |
| 2543 | logger.debug("Cleaning up for Device Testing {}".format(self.instance.build_dir)) |
| 2544 | |
Kumar Gala | b2c07e4 | 2020-10-01 07:25:50 -0500 | [diff] [blame] | 2545 | sanitizelist = [ |
Kumar Gala | 1285c1f | 2020-09-24 13:21:07 -0500 | [diff] [blame] | 2546 | 'CMakeCache.txt', |
| 2547 | 'zephyr/runners.yaml', |
Kumar Gala | b2c07e4 | 2020-10-01 07:25:50 -0500 | [diff] [blame] | 2548 | ] |
| 2549 | keep = [ |
Kumar Gala | 1285c1f | 2020-09-24 13:21:07 -0500 | [diff] [blame] | 2550 | 'zephyr/zephyr.hex', |
| 2551 | 'zephyr/zephyr.bin', |
| 2552 | 'zephyr/zephyr.elf', |
| 2553 | ] |
| 2554 | |
Kumar Gala | b2c07e4 | 2020-10-01 07:25:50 -0500 | [diff] [blame] | 2555 | keep += sanitizelist |
| 2556 | |
Kumar Gala | 1285c1f | 2020-09-24 13:21:07 -0500 | [diff] [blame] | 2557 | self.cleanup_artifacts(keep) |
| 2558 | |
Kumar Gala | b2c07e4 | 2020-10-01 07:25:50 -0500 | [diff] [blame] | 2559 | # sanitize paths so files are relocatable |
| 2560 | for file in sanitizelist: |
| 2561 | file = os.path.join(self.instance.build_dir, file) |
| 2562 | |
| 2563 | with open(file, "rt") as fin: |
| 2564 | data = fin.read() |
| 2565 | data = data.replace(canonical_zephyr_base+"/", "") |
| 2566 | |
| 2567 | with open(file, "wt") as fin: |
| 2568 | fin.write(data) |
| 2569 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2570 | def report_out(self, results): |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2571 | total_to_do = results.total - results.skipped_configs |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2572 | total_tests_width = len(str(total_to_do)) |
| 2573 | results.done += 1 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2574 | instance = self.instance |
| 2575 | |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 2576 | if instance.status in ["error", "failed", "timeout", "flash_error"]: |
Anas Nashif | dc43c29 | 2020-07-09 09:46:45 -0400 | [diff] [blame] | 2577 | if instance.status == "error": |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2578 | results.error += 1 |
| 2579 | results.failed += 1 |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 2580 | if self.verbose: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2581 | status = Fore.RED + "FAILED " + Fore.RESET + instance.reason |
| 2582 | else: |
| 2583 | print("") |
| 2584 | logger.error( |
| 2585 | "{:<25} {:<50} {}FAILED{}: {}".format( |
| 2586 | instance.platform.name, |
| 2587 | instance.testcase.name, |
| 2588 | Fore.RED, |
| 2589 | Fore.RESET, |
| 2590 | instance.reason)) |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 2591 | if not self.verbose: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2592 | self.log_info_file(self.inline_logs) |
| 2593 | elif instance.status == "skipped": |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2594 | status = Fore.YELLOW + "SKIPPED" + Fore.RESET |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 2595 | elif instance.status == "passed": |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2596 | status = Fore.GREEN + "PASSED" + Fore.RESET |
Anas Nashif | 869ca05 | 2020-07-07 14:29:07 -0400 | [diff] [blame] | 2597 | else: |
| 2598 | logger.debug(f"Unknown status = {instance.status}") |
| 2599 | status = Fore.YELLOW + "UNKNOWN" + Fore.RESET |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2600 | |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 2601 | if self.verbose: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2602 | if self.cmake_only: |
| 2603 | more_info = "cmake" |
| 2604 | elif instance.status == "skipped": |
| 2605 | more_info = instance.reason |
| 2606 | else: |
| 2607 | if instance.handler and instance.run: |
| 2608 | more_info = instance.handler.type_str |
| 2609 | htime = instance.handler.duration |
| 2610 | if htime: |
| 2611 | more_info += " {:.3f}s".format(htime) |
| 2612 | else: |
| 2613 | more_info = "build" |
| 2614 | |
| 2615 | logger.info("{:>{}}/{} {:<25} {:<50} {} ({})".format( |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2616 | results.done, total_tests_width, total_to_do, instance.platform.name, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2617 | instance.testcase.name, status, more_info)) |
| 2618 | |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 2619 | if instance.status in ["error", "failed", "timeout"]: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2620 | self.log_info_file(self.inline_logs) |
| 2621 | else: |
Maciej Perkowski | 2ec5ec2 | 2020-09-28 12:37:55 +0200 | [diff] [blame] | 2622 | completed_perc = 0 |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2623 | if total_to_do > 0: |
| 2624 | completed_perc = int((float(results.done) / total_to_do) * 100) |
Maciej Perkowski | 2ec5ec2 | 2020-09-28 12:37:55 +0200 | [diff] [blame] | 2625 | |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2626 | skipped = results.skipped_configs + results.skipped_runtime |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2627 | sys.stdout.write("\rINFO - Total complete: %s%4d/%4d%s %2d%% skipped: %s%4d%s, failed: %s%4d%s" % ( |
| 2628 | Fore.GREEN, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2629 | results.done, |
| 2630 | total_to_do, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2631 | Fore.RESET, |
Maciej Perkowski | 2ec5ec2 | 2020-09-28 12:37:55 +0200 | [diff] [blame] | 2632 | completed_perc, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2633 | Fore.YELLOW if skipped > 0 else Fore.RESET, |
| 2634 | skipped, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2635 | Fore.RESET, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2636 | Fore.RED if results.failed > 0 else Fore.RESET, |
| 2637 | results.failed, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2638 | Fore.RESET |
| 2639 | ) |
| 2640 | ) |
| 2641 | sys.stdout.flush() |
| 2642 | |
| 2643 | def cmake(self): |
| 2644 | |
| 2645 | instance = self.instance |
| 2646 | args = self.testcase.extra_args[:] |
| 2647 | args += self.extra_args |
| 2648 | |
| 2649 | if instance.handler: |
| 2650 | args += instance.handler.args |
| 2651 | |
| 2652 | # merge overlay files into one variable |
| 2653 | def extract_overlays(args): |
| 2654 | re_overlay = re.compile('OVERLAY_CONFIG=(.*)') |
| 2655 | other_args = [] |
| 2656 | overlays = [] |
| 2657 | for arg in args: |
| 2658 | match = re_overlay.search(arg) |
| 2659 | if match: |
| 2660 | overlays.append(match.group(1).strip('\'"')) |
| 2661 | else: |
| 2662 | other_args.append(arg) |
| 2663 | |
| 2664 | args[:] = other_args |
| 2665 | return overlays |
| 2666 | |
| 2667 | overlays = extract_overlays(args) |
| 2668 | |
Torsten Rasmussen | ca08cc0 | 2020-11-25 21:56:46 +0100 | [diff] [blame] | 2669 | if os.path.exists(os.path.join(instance.build_dir, |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 2670 | "twister", "testcase_extra.conf")): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2671 | overlays.append(os.path.join(instance.build_dir, |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 2672 | "twister", "testcase_extra.conf")) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2673 | |
| 2674 | if overlays: |
| 2675 | args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays))) |
| 2676 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2677 | res = self.run_cmake(args) |
| 2678 | return res |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2679 | |
| 2680 | def build(self): |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2681 | res = self.run_build(['--build', self.build_dir]) |
| 2682 | return res |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2683 | |
| 2684 | def run(self): |
| 2685 | |
| 2686 | instance = self.instance |
| 2687 | |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 2688 | if instance.handler: |
| 2689 | if instance.handler.type_str == "device": |
| 2690 | instance.handler.suite = self.suite |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2691 | |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 2692 | instance.handler.handle() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2693 | |
| 2694 | sys.stdout.flush() |
| 2695 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 2696 | class TestSuite(DisablePyTestCollectionMixin): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2697 | config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') |
| 2698 | dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') |
| 2699 | |
| 2700 | tc_schema = scl.yaml_load( |
| 2701 | os.path.join(ZEPHYR_BASE, |
Anas Nashif | c24bf6f | 2020-12-07 11:18:07 -0500 | [diff] [blame] | 2702 | "scripts", "schemas", "twister", "testcase-schema.yaml")) |
Maciej Perkowski | 9c6dfce | 2021-03-11 13:18:33 +0100 | [diff] [blame] | 2703 | quarantine_schema = scl.yaml_load( |
| 2704 | os.path.join(ZEPHYR_BASE, |
| 2705 | "scripts", "schemas", "twister", "quarantine-schema.yaml")) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2706 | |
| 2707 | testcase_valid_keys = {"tags": {"type": "set", "required": False}, |
| 2708 | "type": {"type": "str", "default": "integration"}, |
| 2709 | "extra_args": {"type": "list"}, |
| 2710 | "extra_configs": {"type": "list"}, |
| 2711 | "build_only": {"type": "bool", "default": False}, |
| 2712 | "build_on_all": {"type": "bool", "default": False}, |
| 2713 | "skip": {"type": "bool", "default": False}, |
| 2714 | "slow": {"type": "bool", "default": False}, |
| 2715 | "timeout": {"type": "int", "default": 60}, |
| 2716 | "min_ram": {"type": "int", "default": 8}, |
| 2717 | "depends_on": {"type": "set"}, |
| 2718 | "min_flash": {"type": "int", "default": 32}, |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 2719 | "arch_allow": {"type": "set"}, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2720 | "arch_exclude": {"type": "set"}, |
| 2721 | "extra_sections": {"type": "list", "default": []}, |
Anas Nashif | 1636c31 | 2020-05-28 08:02:54 -0400 | [diff] [blame] | 2722 | "integration_platforms": {"type": "list", "default": []}, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2723 | "platform_exclude": {"type": "set"}, |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 2724 | "platform_allow": {"type": "set"}, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2725 | "toolchain_exclude": {"type": "set"}, |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 2726 | "toolchain_allow": {"type": "set"}, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2727 | "filter": {"type": "str"}, |
| 2728 | "harness": {"type": "str"}, |
| 2729 | "harness_config": {"type": "map", "default": {}} |
| 2730 | } |
| 2731 | |
Anas Nashif | 94f6826 | 2020-12-07 11:21:04 -0500 | [diff] [blame] | 2732 | RELEASE_DATA = os.path.join(ZEPHYR_BASE, "scripts", "release", |
| 2733 | "twister_last_release.csv") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2734 | |
Aastha Grover | a0ae534 | 2020-05-13 13:34:00 -0700 | [diff] [blame] | 2735 | SAMPLE_FILENAME = 'sample.yaml' |
| 2736 | TESTCASE_FILENAME = 'testcase.yaml' |
| 2737 | |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 2738 | def __init__(self, board_root_list=[], testcase_roots=[], outdir=None): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2739 | |
| 2740 | self.roots = testcase_roots |
| 2741 | if not isinstance(board_root_list, list): |
| 2742 | self.board_roots = [board_root_list] |
| 2743 | else: |
| 2744 | self.board_roots = board_root_list |
| 2745 | |
| 2746 | # Testsuite Options |
| 2747 | self.coverage_platform = [] |
| 2748 | self.build_only = False |
| 2749 | self.cmake_only = False |
| 2750 | self.cleanup = False |
| 2751 | self.enable_slow = False |
| 2752 | self.device_testing = False |
Anas Nashif | ce8c12e | 2020-05-21 09:11:40 -0400 | [diff] [blame] | 2753 | self.fixtures = [] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2754 | self.enable_coverage = False |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 2755 | self.enable_ubsan = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2756 | self.enable_lsan = False |
| 2757 | self.enable_asan = False |
| 2758 | self.enable_valgrind = False |
| 2759 | self.extra_args = [] |
| 2760 | self.inline_logs = False |
| 2761 | self.enable_sizes_report = False |
| 2762 | self.west_flash = None |
| 2763 | self.west_runner = None |
| 2764 | self.generator = None |
| 2765 | self.generator_cmd = None |
Anas Nashif | 5092541 | 2020-07-16 17:25:19 -0400 | [diff] [blame] | 2766 | self.warnings_as_errors = True |
Anas Nashif | f68146f | 2020-11-28 11:07:06 -0500 | [diff] [blame] | 2767 | self.overflow_as_errors = False |
Maciej Perkowski | 9c6dfce | 2021-03-11 13:18:33 +0100 | [diff] [blame] | 2768 | self.quarantine_verify = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2769 | |
| 2770 | # Keep track of which test cases we've filtered out and why |
| 2771 | self.testcases = {} |
Maciej Perkowski | 9c6dfce | 2021-03-11 13:18:33 +0100 | [diff] [blame] | 2772 | self.quarantine = {} |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2773 | self.platforms = [] |
| 2774 | self.selected_platforms = [] |
Maciej Perkowski | b6a69aa | 2021-02-04 16:05:21 +0100 | [diff] [blame] | 2775 | self.filtered_platforms = [] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2776 | self.default_platforms = [] |
| 2777 | self.outdir = os.path.abspath(outdir) |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 2778 | self.discards = {} |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2779 | self.load_errors = 0 |
| 2780 | self.instances = dict() |
| 2781 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2782 | self.total_platforms = 0 |
| 2783 | self.start_time = 0 |
| 2784 | self.duration = 0 |
| 2785 | self.warnings = 0 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2786 | |
| 2787 | # hardcoded for now |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 2788 | self.duts = [] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2789 | |
Anas Nashif | 1636c31 | 2020-05-28 08:02:54 -0400 | [diff] [blame] | 2790 | # run integration tests only |
| 2791 | self.integration = False |
| 2792 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2793 | self.pipeline = None |
Maciej Perkowski | 060e00d | 2020-10-13 18:59:37 +0200 | [diff] [blame] | 2794 | self.version = "NA" |
| 2795 | |
| 2796 | def check_zephyr_version(self): |
| 2797 | try: |
Jingru Wang | b9c953c | 2020-12-21 10:45:27 +0800 | [diff] [blame] | 2798 | subproc = subprocess.run(["git", "describe", "--abbrev=12"], |
Maciej Perkowski | 060e00d | 2020-10-13 18:59:37 +0200 | [diff] [blame] | 2799 | stdout=subprocess.PIPE, |
| 2800 | universal_newlines=True, |
| 2801 | cwd=ZEPHYR_BASE) |
| 2802 | if subproc.returncode == 0: |
| 2803 | self.version = subproc.stdout.strip() |
| 2804 | logger.info(f"Zephyr version: {self.version}") |
| 2805 | except OSError: |
| 2806 | logger.info("Cannot read zephyr version.") |
| 2807 | |
Anas Nashif | bb28035 | 2020-05-07 12:02:48 -0400 | [diff] [blame] | 2808 | def get_platform_instances(self, platform): |
Piotr Golyzniak | 8b77348 | 2021-11-02 16:13:26 +0100 | [diff] [blame] | 2809 | filtered_dict = {k:v for k,v in self.instances.items() if k.startswith(platform + os.sep)} |
Anas Nashif | bb28035 | 2020-05-07 12:02:48 -0400 | [diff] [blame] | 2810 | return filtered_dict |
| 2811 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2812 | def config(self): |
| 2813 | logger.info("coverage platform: {}".format(self.coverage_platform)) |
| 2814 | |
| 2815 | # Debug Functions |
| 2816 | @staticmethod |
| 2817 | def info(what): |
| 2818 | sys.stdout.write(what + "\n") |
| 2819 | sys.stdout.flush() |
| 2820 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2821 | def update_counting(self, results=None, initial=False): |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2822 | results.skipped_configs = 0 |
| 2823 | results.skipped_cases = 0 |
Maciej Perkowski | c67a0cd | 2020-08-13 15:20:13 +0200 | [diff] [blame] | 2824 | for instance in self.instances.values(): |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2825 | if initial: |
| 2826 | results.cases += len(instance.testcase.cases) |
Maciej Perkowski | c67a0cd | 2020-08-13 15:20:13 +0200 | [diff] [blame] | 2827 | if instance.status == 'skipped': |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2828 | results.skipped_configs += 1 |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2829 | results.skipped_cases += len(instance.testcase.cases) |
Maciej Perkowski | c67a0cd | 2020-08-13 15:20:13 +0200 | [diff] [blame] | 2830 | elif instance.status == "passed": |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2831 | results.passed += 1 |
Maciej Perkowski | c67a0cd | 2020-08-13 15:20:13 +0200 | [diff] [blame] | 2832 | for res in instance.results.values(): |
| 2833 | if res == 'SKIP': |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2834 | results.skipped_cases += 1 |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 2835 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2836 | def compare_metrics(self, filename): |
| 2837 | # name, datatype, lower results better |
| 2838 | interesting_metrics = [("ram_size", int, True), |
| 2839 | ("rom_size", int, True)] |
| 2840 | |
| 2841 | if not os.path.exists(filename): |
Anas Nashif | ad44bed | 2020-08-24 18:59:01 -0400 | [diff] [blame] | 2842 | logger.error("Cannot compare metrics, %s not found" % filename) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2843 | return [] |
| 2844 | |
| 2845 | results = [] |
| 2846 | saved_metrics = {} |
| 2847 | with open(filename) as fp: |
| 2848 | cr = csv.DictReader(fp) |
| 2849 | for row in cr: |
| 2850 | d = {} |
| 2851 | for m, _, _ in interesting_metrics: |
| 2852 | d[m] = row[m] |
| 2853 | saved_metrics[(row["test"], row["platform"])] = d |
| 2854 | |
| 2855 | for instance in self.instances.values(): |
| 2856 | mkey = (instance.testcase.name, instance.platform.name) |
| 2857 | if mkey not in saved_metrics: |
| 2858 | continue |
| 2859 | sm = saved_metrics[mkey] |
| 2860 | for metric, mtype, lower_better in interesting_metrics: |
| 2861 | if metric not in instance.metrics: |
| 2862 | continue |
| 2863 | if sm[metric] == "": |
| 2864 | continue |
| 2865 | delta = instance.metrics.get(metric, 0) - mtype(sm[metric]) |
| 2866 | if delta == 0: |
| 2867 | continue |
| 2868 | results.append((instance, metric, instance.metrics.get(metric, 0), delta, |
| 2869 | lower_better)) |
| 2870 | return results |
| 2871 | |
Anas Nashif | ad44bed | 2020-08-24 18:59:01 -0400 | [diff] [blame] | 2872 | def footprint_reports(self, report, show_footprint, all_deltas, |
| 2873 | footprint_threshold, last_metrics): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2874 | if not report: |
| 2875 | return |
| 2876 | |
Anas Nashif | ad44bed | 2020-08-24 18:59:01 -0400 | [diff] [blame] | 2877 | logger.debug("running footprint_reports") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2878 | deltas = self.compare_metrics(report) |
| 2879 | warnings = 0 |
| 2880 | if deltas and show_footprint: |
| 2881 | for i, metric, value, delta, lower_better in deltas: |
| 2882 | if not all_deltas and ((delta < 0 and lower_better) or |
| 2883 | (delta > 0 and not lower_better)): |
| 2884 | continue |
| 2885 | |
Anas Nashif | ad44bed | 2020-08-24 18:59:01 -0400 | [diff] [blame] | 2886 | percentage = 0 |
| 2887 | if value > delta: |
| 2888 | percentage = (float(delta) / float(value - delta)) |
| 2889 | |
| 2890 | if not all_deltas and (percentage < (footprint_threshold / 100.0)): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2891 | continue |
| 2892 | |
| 2893 | logger.info("{:<25} {:<60} {}{}{}: {} {:<+4}, is now {:6} {:+.2%}".format( |
| 2894 | i.platform.name, i.testcase.name, Fore.YELLOW, |
| 2895 | "INFO" if all_deltas else "WARNING", Fore.RESET, |
| 2896 | metric, delta, value, percentage)) |
| 2897 | warnings += 1 |
| 2898 | |
| 2899 | if warnings: |
| 2900 | logger.warning("Deltas based on metrics from last %s" % |
| 2901 | ("release" if not last_metrics else "run")) |
| 2902 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2903 | def summary(self, results, unrecognized_sections): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2904 | failed = 0 |
Anas Nashif | 4258d8d | 2020-05-08 08:40:27 -0400 | [diff] [blame] | 2905 | run = 0 |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2906 | for instance in self.instances.values(): |
| 2907 | if instance.status == "failed": |
| 2908 | failed += 1 |
| 2909 | elif instance.metrics.get("unrecognized") and not unrecognized_sections: |
| 2910 | logger.error("%sFAILED%s: %s has unrecognized binary sections: %s" % |
| 2911 | (Fore.RED, Fore.RESET, instance.name, |
| 2912 | str(instance.metrics.get("unrecognized", [])))) |
| 2913 | failed += 1 |
| 2914 | |
Anas Nashif | ad44bed | 2020-08-24 18:59:01 -0400 | [diff] [blame] | 2915 | if instance.metrics.get('handler_time', None): |
Anas Nashif | 4258d8d | 2020-05-08 08:40:27 -0400 | [diff] [blame] | 2916 | run += 1 |
| 2917 | |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2918 | if results.total and results.total != results.skipped_configs: |
| 2919 | pass_rate = (float(results.passed) / float(results.total - results.skipped_configs)) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2920 | else: |
| 2921 | pass_rate = 0 |
| 2922 | |
| 2923 | logger.info( |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2924 | "{}{} of {}{} test configurations passed ({:.2%}), {}{}{} failed, {} skipped with {}{}{} warnings in {:.2f} seconds".format( |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2925 | Fore.RED if failed else Fore.GREEN, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2926 | results.passed, |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2927 | results.total - results.skipped_configs, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2928 | Fore.RESET, |
| 2929 | pass_rate, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2930 | Fore.RED if results.failed else Fore.RESET, |
| 2931 | results.failed, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2932 | Fore.RESET, |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2933 | results.skipped_configs, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2934 | Fore.YELLOW if self.warnings else Fore.RESET, |
| 2935 | self.warnings, |
| 2936 | Fore.RESET, |
| 2937 | self.duration)) |
| 2938 | |
| 2939 | self.total_platforms = len(self.platforms) |
Anas Nashif | bb42795 | 2020-11-24 08:39:42 -0500 | [diff] [blame] | 2940 | # if we are only building, do not report about tests being executed. |
| 2941 | if self.platforms and not self.build_only: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 2942 | logger.info("In total {} test cases were executed, {} skipped on {} out of total {} platforms ({:02.2f}%)".format( |
| 2943 | results.cases - results.skipped_cases, |
| 2944 | results.skipped_cases, |
Maciej Perkowski | b6a69aa | 2021-02-04 16:05:21 +0100 | [diff] [blame] | 2945 | len(self.filtered_platforms), |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2946 | self.total_platforms, |
Maciej Perkowski | b6a69aa | 2021-02-04 16:05:21 +0100 | [diff] [blame] | 2947 | (100 * len(self.filtered_platforms) / len(self.platforms)) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2948 | )) |
| 2949 | |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 2950 | logger.info(f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms, \ |
| 2951 | {Fore.RED}{results.total - run - results.skipped_configs}{Fore.RESET} test configurations were only built.") |
Anas Nashif | 4258d8d | 2020-05-08 08:40:27 -0400 | [diff] [blame] | 2952 | |
Anas Nashif | ec47912 | 2021-01-25 08:13:27 -0500 | [diff] [blame] | 2953 | def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2954 | if not self.instances: |
| 2955 | return |
| 2956 | |
Anas Nashif | a5d16ab | 2020-12-10 11:45:57 -0500 | [diff] [blame] | 2957 | logger.info("Saving reports...") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2958 | if name: |
| 2959 | report_name = name |
| 2960 | else: |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 2961 | report_name = "twister" |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2962 | |
| 2963 | if report_dir: |
| 2964 | os.makedirs(report_dir, exist_ok=True) |
| 2965 | filename = os.path.join(report_dir, report_name) |
| 2966 | outdir = report_dir |
| 2967 | else: |
| 2968 | filename = os.path.join(self.outdir, report_name) |
| 2969 | outdir = self.outdir |
| 2970 | |
Anas Nashif | 6915adf | 2020-04-22 09:39:42 -0400 | [diff] [blame] | 2971 | if suffix: |
| 2972 | filename = "{}_{}".format(filename, suffix) |
| 2973 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2974 | if not no_update: |
Maciej Perkowski | 060e00d | 2020-10-13 18:59:37 +0200 | [diff] [blame] | 2975 | self.xunit_report(filename + ".xml", full_report=False, |
| 2976 | append=only_failed, version=self.version) |
| 2977 | self.xunit_report(filename + "_report.xml", full_report=True, |
| 2978 | append=only_failed, version=self.version) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2979 | self.csv_report(filename + ".csv") |
Anas Nashif | ec47912 | 2021-01-25 08:13:27 -0500 | [diff] [blame] | 2980 | |
| 2981 | if json_report: |
| 2982 | self.json_report(filename + ".json", append=only_failed, version=self.version) |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 2983 | |
Anas Nashif | a5d16ab | 2020-12-10 11:45:57 -0500 | [diff] [blame] | 2984 | if platform_reports: |
| 2985 | self.target_report(outdir, suffix, append=only_failed) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 2986 | if self.discards: |
| 2987 | self.discard_report(filename + "_discard.csv") |
| 2988 | |
| 2989 | if release: |
| 2990 | self.csv_report(self.RELEASE_DATA) |
| 2991 | |
| 2992 | def add_configurations(self): |
| 2993 | |
| 2994 | for board_root in self.board_roots: |
| 2995 | board_root = os.path.abspath(board_root) |
| 2996 | |
| 2997 | logger.debug("Reading platform configuration files under %s..." % |
| 2998 | board_root) |
| 2999 | |
| 3000 | for file in glob.glob(os.path.join(board_root, "*", "*", "*.yaml")): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3001 | try: |
| 3002 | platform = Platform() |
| 3003 | platform.load(file) |
Anas Nashif | 61c4a51 | 2020-08-13 07:46:45 -0400 | [diff] [blame] | 3004 | if platform.name in [p.name for p in self.platforms]: |
| 3005 | logger.error(f"Duplicate platform {platform.name} in {file}") |
| 3006 | raise Exception(f"Duplicate platform identifier {platform.name} found") |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 3007 | if platform.twister: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3008 | self.platforms.append(platform) |
| 3009 | if platform.default: |
| 3010 | self.default_platforms.append(platform.name) |
| 3011 | |
| 3012 | except RuntimeError as e: |
| 3013 | logger.error("E: %s: can't load: %s" % (file, e)) |
| 3014 | self.load_errors += 1 |
| 3015 | |
| 3016 | def get_all_tests(self): |
| 3017 | tests = [] |
| 3018 | for _, tc in self.testcases.items(): |
| 3019 | for case in tc.cases: |
| 3020 | tests.append(case) |
| 3021 | |
| 3022 | return tests |
| 3023 | |
| 3024 | @staticmethod |
| 3025 | def get_toolchain(): |
Torsten Rasmussen | a088970 | 2021-02-05 09:55:41 +0100 | [diff] [blame] | 3026 | toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake') |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 3027 | result = CMake.run_cmake_script([toolchain_script, "FORMAT=json"]) |
| 3028 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3029 | try: |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 3030 | if result['returncode']: |
Torsten Rasmussen | af87599 | 2021-09-28 13:04:52 +0200 | [diff] [blame] | 3031 | raise TwisterRuntimeError(f"E: {result['returnmsg']}") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3032 | except Exception as e: |
| 3033 | print(str(e)) |
| 3034 | sys.exit(2) |
Torsten Rasmussen | d162e9e | 2021-02-04 15:55:23 +0100 | [diff] [blame] | 3035 | toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT'] |
| 3036 | logger.info(f"Using '{toolchain}' toolchain.") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3037 | |
| 3038 | return toolchain |
| 3039 | |
| 3040 | def add_testcases(self, testcase_filter=[]): |
| 3041 | for root in self.roots: |
| 3042 | root = os.path.abspath(root) |
| 3043 | |
| 3044 | logger.debug("Reading test case configuration files under %s..." % root) |
| 3045 | |
Jorgen Kvalvaag | 36d96d2 | 2021-04-06 14:57:28 +0200 | [diff] [blame] | 3046 | for dirpath, _, filenames in os.walk(root, topdown=True): |
Aastha Grover | a0ae534 | 2020-05-13 13:34:00 -0700 | [diff] [blame] | 3047 | if self.SAMPLE_FILENAME in filenames: |
| 3048 | filename = self.SAMPLE_FILENAME |
| 3049 | elif self.TESTCASE_FILENAME in filenames: |
| 3050 | filename = self.TESTCASE_FILENAME |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3051 | else: |
| 3052 | continue |
| 3053 | |
| 3054 | logger.debug("Found possible test case in " + dirpath) |
| 3055 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3056 | tc_path = os.path.join(dirpath, filename) |
| 3057 | |
| 3058 | try: |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 3059 | parsed_data = TwisterConfigParser(tc_path, self.tc_schema) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3060 | parsed_data.load() |
| 3061 | |
| 3062 | tc_path = os.path.dirname(tc_path) |
| 3063 | workdir = os.path.relpath(tc_path, root) |
| 3064 | |
| 3065 | for name in parsed_data.tests.keys(): |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 3066 | tc = TestCase(root, workdir, name) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3067 | |
| 3068 | tc_dict = parsed_data.get_test(name, self.testcase_valid_keys) |
| 3069 | |
| 3070 | tc.source_dir = tc_path |
| 3071 | tc.yamlfile = tc_path |
| 3072 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3073 | tc.type = tc_dict["type"] |
| 3074 | tc.tags = tc_dict["tags"] |
| 3075 | tc.extra_args = tc_dict["extra_args"] |
| 3076 | tc.extra_configs = tc_dict["extra_configs"] |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3077 | tc.arch_allow = tc_dict["arch_allow"] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3078 | tc.arch_exclude = tc_dict["arch_exclude"] |
| 3079 | tc.skip = tc_dict["skip"] |
| 3080 | tc.platform_exclude = tc_dict["platform_exclude"] |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3081 | tc.platform_allow = tc_dict["platform_allow"] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3082 | tc.toolchain_exclude = tc_dict["toolchain_exclude"] |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3083 | tc.toolchain_allow = tc_dict["toolchain_allow"] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3084 | tc.tc_filter = tc_dict["filter"] |
| 3085 | tc.timeout = tc_dict["timeout"] |
| 3086 | tc.harness = tc_dict["harness"] |
| 3087 | tc.harness_config = tc_dict["harness_config"] |
Anas Nashif | 43275c8 | 2020-05-04 18:22:16 -0400 | [diff] [blame] | 3088 | if tc.harness == 'console' and not tc.harness_config: |
| 3089 | raise Exception('Harness config error: console harness defined without a configuration.') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3090 | tc.build_only = tc_dict["build_only"] |
| 3091 | tc.build_on_all = tc_dict["build_on_all"] |
| 3092 | tc.slow = tc_dict["slow"] |
| 3093 | tc.min_ram = tc_dict["min_ram"] |
| 3094 | tc.depends_on = tc_dict["depends_on"] |
| 3095 | tc.min_flash = tc_dict["min_flash"] |
| 3096 | tc.extra_sections = tc_dict["extra_sections"] |
Anas Nashif | 1636c31 | 2020-05-28 08:02:54 -0400 | [diff] [blame] | 3097 | tc.integration_platforms = tc_dict["integration_platforms"] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3098 | |
| 3099 | tc.parse_subcases(tc_path) |
| 3100 | |
| 3101 | if testcase_filter: |
| 3102 | if tc.name and tc.name in testcase_filter: |
| 3103 | self.testcases[tc.name] = tc |
| 3104 | else: |
| 3105 | self.testcases[tc.name] = tc |
| 3106 | |
| 3107 | except Exception as e: |
| 3108 | logger.error("%s: can't load (skipping): %s" % (tc_path, e)) |
| 3109 | self.load_errors += 1 |
Anas Nashif | fe07d57 | 2020-11-20 12:13:47 -0500 | [diff] [blame] | 3110 | return len(self.testcases) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3111 | |
| 3112 | def get_platform(self, name): |
| 3113 | selected_platform = None |
| 3114 | for platform in self.platforms: |
| 3115 | if platform.name == name: |
| 3116 | selected_platform = platform |
| 3117 | break |
| 3118 | return selected_platform |
| 3119 | |
Maciej Perkowski | 9c6dfce | 2021-03-11 13:18:33 +0100 | [diff] [blame] | 3120 | def load_quarantine(self, file): |
| 3121 | """ |
| 3122 | Loads quarantine list from the given yaml file. Creates a dictionary |
| 3123 | of all tests configurations (platform + scenario: comment) that shall be |
| 3124 | skipped due to quarantine |
| 3125 | """ |
| 3126 | |
| 3127 | # Load yaml into quarantine_yaml |
| 3128 | quarantine_yaml = scl.yaml_load_verify(file, self.quarantine_schema) |
| 3129 | |
| 3130 | # Create quarantine_list with a product of the listed |
| 3131 | # platforms and scenarios for each entry in quarantine yaml |
| 3132 | quarantine_list = [] |
| 3133 | for quar_dict in quarantine_yaml: |
| 3134 | if quar_dict['platforms'][0] == "all": |
| 3135 | plat = [p.name for p in self.platforms] |
| 3136 | else: |
| 3137 | plat = quar_dict['platforms'] |
| 3138 | comment = quar_dict.get('comment', "NA") |
| 3139 | quarantine_list.append([{".".join([p, s]): comment} |
| 3140 | for p in plat for s in quar_dict['scenarios']]) |
| 3141 | |
| 3142 | # Flatten the quarantine_list |
| 3143 | quarantine_list = [it for sublist in quarantine_list for it in sublist] |
| 3144 | # Change quarantine_list into a dictionary |
| 3145 | for d in quarantine_list: |
| 3146 | self.quarantine.update(d) |
| 3147 | |
Anas Nashif | 82a6a46 | 2020-11-02 10:47:36 -0500 | [diff] [blame] | 3148 | def load_from_file(self, file, filter_status=[], filter_platform=[]): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3149 | try: |
| 3150 | with open(file, "r") as fp: |
| 3151 | cr = csv.DictReader(fp) |
| 3152 | instance_list = [] |
| 3153 | for row in cr: |
| 3154 | if row["status"] in filter_status: |
| 3155 | continue |
| 3156 | test = row["test"] |
| 3157 | |
| 3158 | platform = self.get_platform(row["platform"]) |
Anas Nashif | 82a6a46 | 2020-11-02 10:47:36 -0500 | [diff] [blame] | 3159 | if filter_platform and platform.name not in filter_platform: |
| 3160 | continue |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3161 | instance = TestInstance(self.testcases[test], platform, self.outdir) |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3162 | if self.device_testing: |
| 3163 | tfilter = 'runnable' |
| 3164 | else: |
| 3165 | tfilter = 'buildable' |
| 3166 | instance.run = instance.check_runnable( |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3167 | self.enable_slow, |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3168 | tfilter, |
Anas Nashif | ce8c12e | 2020-05-21 09:11:40 -0400 | [diff] [blame] | 3169 | self.fixtures |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3170 | ) |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 3171 | instance.create_overlay(platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3172 | instance_list.append(instance) |
| 3173 | self.add_instances(instance_list) |
| 3174 | |
| 3175 | except KeyError as e: |
| 3176 | logger.error("Key error while parsing tests file.({})".format(str(e))) |
| 3177 | sys.exit(2) |
| 3178 | |
| 3179 | except FileNotFoundError as e: |
| 3180 | logger.error("Couldn't find input file with list of tests. ({})".format(e)) |
| 3181 | sys.exit(2) |
| 3182 | |
| 3183 | def apply_filters(self, **kwargs): |
| 3184 | |
| 3185 | toolchain = self.get_toolchain() |
| 3186 | |
| 3187 | discards = {} |
| 3188 | platform_filter = kwargs.get('platform') |
Anas Nashif | aff616d | 2020-04-17 21:24:57 -0400 | [diff] [blame] | 3189 | exclude_platform = kwargs.get('exclude_platform', []) |
| 3190 | testcase_filter = kwargs.get('run_individual_tests', []) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3191 | arch_filter = kwargs.get('arch') |
| 3192 | tag_filter = kwargs.get('tag') |
| 3193 | exclude_tag = kwargs.get('exclude_tag') |
| 3194 | all_filter = kwargs.get('all') |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3195 | runnable = kwargs.get('runnable') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3196 | force_toolchain = kwargs.get('force_toolchain') |
Anas Nashif | 1a5defa | 2020-05-01 14:57:00 -0400 | [diff] [blame] | 3197 | force_platform = kwargs.get('force_platform') |
Anas Nashif | 9eb9c4c | 2020-08-26 15:47:25 -0400 | [diff] [blame] | 3198 | emu_filter = kwargs.get('emulation_only') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3199 | |
| 3200 | logger.debug("platform filter: " + str(platform_filter)) |
| 3201 | logger.debug(" arch_filter: " + str(arch_filter)) |
| 3202 | logger.debug(" tag_filter: " + str(tag_filter)) |
| 3203 | logger.debug(" exclude_tag: " + str(exclude_tag)) |
| 3204 | |
| 3205 | default_platforms = False |
Anas Nashif | 9eb9c4c | 2020-08-26 15:47:25 -0400 | [diff] [blame] | 3206 | emulation_platforms = False |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3207 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3208 | |
| 3209 | if all_filter: |
| 3210 | logger.info("Selecting all possible platforms per test case") |
| 3211 | # When --all used, any --platform arguments ignored |
| 3212 | platform_filter = [] |
Anas Nashif | 9eb9c4c | 2020-08-26 15:47:25 -0400 | [diff] [blame] | 3213 | elif not platform_filter and not emu_filter: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3214 | logger.info("Selecting default platforms per test case") |
| 3215 | default_platforms = True |
Anas Nashif | 9eb9c4c | 2020-08-26 15:47:25 -0400 | [diff] [blame] | 3216 | elif emu_filter: |
| 3217 | logger.info("Selecting emulation platforms per test case") |
| 3218 | emulation_platforms = True |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3219 | |
Anas Nashif | 3b939da | 2020-11-24 13:21:27 -0500 | [diff] [blame] | 3220 | if platform_filter: |
| 3221 | platforms = list(filter(lambda p: p.name in platform_filter, self.platforms)) |
| 3222 | elif emu_filter: |
| 3223 | platforms = list(filter(lambda p: p.simulation != 'na', self.platforms)) |
| 3224 | elif arch_filter: |
| 3225 | platforms = list(filter(lambda p: p.arch in arch_filter, self.platforms)) |
| 3226 | elif default_platforms: |
| 3227 | platforms = list(filter(lambda p: p.default, self.platforms)) |
| 3228 | else: |
| 3229 | platforms = self.platforms |
| 3230 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3231 | logger.info("Building initial testcase list...") |
| 3232 | |
| 3233 | for tc_name, tc in self.testcases.items(): |
Anas Nashif | 827ecb7 | 2021-01-14 09:34:29 -0500 | [diff] [blame] | 3234 | |
| 3235 | if tc.build_on_all and not platform_filter: |
| 3236 | platform_scope = self.platforms |
Anas Nashif | e618a59 | 2021-03-03 14:05:54 -0500 | [diff] [blame] | 3237 | elif tc.integration_platforms and self.integration: |
| 3238 | platform_scope = list(filter(lambda item: item.name in tc.integration_platforms, \ |
| 3239 | self.platforms)) |
Anas Nashif | 827ecb7 | 2021-01-14 09:34:29 -0500 | [diff] [blame] | 3240 | else: |
| 3241 | platform_scope = platforms |
| 3242 | |
Kumar Gala | d590e0d | 2021-04-02 10:18:01 -0500 | [diff] [blame] | 3243 | integration = self.integration and tc.integration_platforms |
| 3244 | |
| 3245 | # If there isn't any overlap between the platform_allow list and the platform_scope |
| 3246 | # we set the scope to the platform_allow list |
| 3247 | if tc.platform_allow and not platform_filter and not integration: |
| 3248 | a = set(platform_scope) |
| 3249 | b = set(filter(lambda item: item.name in tc.platform_allow, self.platforms)) |
| 3250 | c = a.intersection(b) |
| 3251 | if not c: |
| 3252 | platform_scope = list(filter(lambda item: item.name in tc.platform_allow, \ |
| 3253 | self.platforms)) |
| 3254 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3255 | # list of instances per testcase, aka configurations. |
| 3256 | instance_list = [] |
Anas Nashif | 827ecb7 | 2021-01-14 09:34:29 -0500 | [diff] [blame] | 3257 | for plat in platform_scope: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3258 | instance = TestInstance(tc, plat, self.outdir) |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3259 | if runnable: |
| 3260 | tfilter = 'runnable' |
| 3261 | else: |
| 3262 | tfilter = 'buildable' |
| 3263 | |
| 3264 | instance.run = instance.check_runnable( |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3265 | self.enable_slow, |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3266 | tfilter, |
Anas Nashif | ce8c12e | 2020-05-21 09:11:40 -0400 | [diff] [blame] | 3267 | self.fixtures |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3268 | ) |
Anas Nashif | f86bc05 | 2020-11-19 16:05:03 -0500 | [diff] [blame] | 3269 | |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3270 | for t in tc.cases: |
| 3271 | instance.results[t] = None |
Anas Nashif | 3b86f13 | 2020-05-21 10:35:33 -0400 | [diff] [blame] | 3272 | |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 3273 | if runnable and self.duts: |
| 3274 | for h in self.duts: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3275 | if h.platform == plat.name: |
| 3276 | if tc.harness_config.get('fixture') in h.fixtures: |
Anas Nashif | 3b86f13 | 2020-05-21 10:35:33 -0400 | [diff] [blame] | 3277 | instance.run = True |
| 3278 | |
Anas Nashif | 1a5defa | 2020-05-01 14:57:00 -0400 | [diff] [blame] | 3279 | if not force_platform and plat.name in exclude_platform: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3280 | discards[instance] = discards.get(instance, "Platform is excluded on command line.") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3281 | |
| 3282 | if (plat.arch == "unit") != (tc.type == "unit"): |
| 3283 | # Discard silently |
| 3284 | continue |
| 3285 | |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3286 | if runnable and not instance.run: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3287 | discards[instance] = discards.get(instance, "Not runnable on device") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3288 | |
Anas Nashif | 1636c31 | 2020-05-28 08:02:54 -0400 | [diff] [blame] | 3289 | if self.integration and tc.integration_platforms and plat.name not in tc.integration_platforms: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3290 | discards[instance] = discards.get(instance, "Not part of integration platforms") |
Anas Nashif | 1636c31 | 2020-05-28 08:02:54 -0400 | [diff] [blame] | 3291 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3292 | if tc.skip: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3293 | discards[instance] = discards.get(instance, "Skip filter") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3294 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3295 | if tag_filter and not tc.tags.intersection(tag_filter): |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3296 | discards[instance] = discards.get(instance, "Command line testcase tag filter") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3297 | |
| 3298 | if exclude_tag and tc.tags.intersection(exclude_tag): |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3299 | discards[instance] = discards.get(instance, "Command line testcase exclude filter") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3300 | |
| 3301 | if testcase_filter and tc_name not in testcase_filter: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3302 | discards[instance] = discards.get(instance, "Testcase name filter") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3303 | |
| 3304 | if arch_filter and plat.arch not in arch_filter: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3305 | discards[instance] = discards.get(instance, "Command line testcase arch filter") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3306 | |
Anas Nashif | 1a5defa | 2020-05-01 14:57:00 -0400 | [diff] [blame] | 3307 | if not force_platform: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3308 | |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3309 | if tc.arch_allow and plat.arch not in tc.arch_allow: |
| 3310 | discards[instance] = discards.get(instance, "Not in test case arch allow list") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3311 | |
Anas Nashif | 1a5defa | 2020-05-01 14:57:00 -0400 | [diff] [blame] | 3312 | if tc.arch_exclude and plat.arch in tc.arch_exclude: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3313 | discards[instance] = discards.get(instance, "In test case arch exclude") |
Anas Nashif | 1a5defa | 2020-05-01 14:57:00 -0400 | [diff] [blame] | 3314 | |
| 3315 | if tc.platform_exclude and plat.name in tc.platform_exclude: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3316 | discards[instance] = discards.get(instance, "In test case platform exclude") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3317 | |
| 3318 | if tc.toolchain_exclude and toolchain in tc.toolchain_exclude: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3319 | discards[instance] = discards.get(instance, "In test case toolchain exclude") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3320 | |
| 3321 | if platform_filter and plat.name not in platform_filter: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3322 | discards[instance] = discards.get(instance, "Command line platform filter") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3323 | |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3324 | if tc.platform_allow and plat.name not in tc.platform_allow: |
| 3325 | discards[instance] = discards.get(instance, "Not in testcase platform allow list") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3326 | |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3327 | if tc.toolchain_allow and toolchain not in tc.toolchain_allow: |
| 3328 | discards[instance] = discards.get(instance, "Not in testcase toolchain allow list") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3329 | |
| 3330 | if not plat.env_satisfied: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3331 | discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env))) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3332 | |
| 3333 | if not force_toolchain \ |
| 3334 | and toolchain and (toolchain not in plat.supported_toolchains) \ |
Maciej Perkowski | edb23d7 | 2021-09-09 16:16:49 +0200 | [diff] [blame] | 3335 | and "host" not in plat.supported_toolchains \ |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3336 | and tc.type != 'unit': |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3337 | discards[instance] = discards.get(instance, "Not supported by the toolchain") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3338 | |
| 3339 | if plat.ram < tc.min_ram: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3340 | discards[instance] = discards.get(instance, "Not enough RAM") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3341 | |
| 3342 | if tc.depends_on: |
| 3343 | dep_intersection = tc.depends_on.intersection(set(plat.supported)) |
| 3344 | if dep_intersection != set(tc.depends_on): |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3345 | discards[instance] = discards.get(instance, "No hardware support") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3346 | |
| 3347 | if plat.flash < tc.min_flash: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3348 | discards[instance] = discards.get(instance, "Not enough FLASH") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3349 | |
| 3350 | if set(plat.ignore_tags) & tc.tags: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3351 | discards[instance] = discards.get(instance, "Excluded tags per platform (exclude_tags)") |
Anas Nashif | e8e367a | 2020-07-16 16:27:04 -0400 | [diff] [blame] | 3352 | |
Anas Nashif | 555fc6d | 2020-07-30 07:23:54 -0400 | [diff] [blame] | 3353 | if plat.only_tags and not set(plat.only_tags) & tc.tags: |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3354 | discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3355 | |
Maciej Perkowski | 9c6dfce | 2021-03-11 13:18:33 +0100 | [diff] [blame] | 3356 | test_configuration = ".".join([instance.platform.name, |
| 3357 | instance.testcase.id]) |
| 3358 | # skip quarantined tests |
| 3359 | if test_configuration in self.quarantine and not self.quarantine_verify: |
| 3360 | discards[instance] = discards.get(instance, |
| 3361 | f"Quarantine: {self.quarantine[test_configuration]}") |
| 3362 | # run only quarantined test to verify their statuses (skip everything else) |
| 3363 | if self.quarantine_verify and test_configuration not in self.quarantine: |
| 3364 | discards[instance] = discards.get(instance, "Not under quarantine") |
| 3365 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3366 | # if nothing stopped us until now, it means this configuration |
| 3367 | # needs to be added. |
| 3368 | instance_list.append(instance) |
| 3369 | |
| 3370 | # no configurations, so jump to next testcase |
| 3371 | if not instance_list: |
| 3372 | continue |
| 3373 | |
Anas Nashif | b18f311 | 2020-12-07 11:40:19 -0500 | [diff] [blame] | 3374 | # if twister was launched with no platform options at all, we |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3375 | # take all default platforms |
Anas Nashif | e618a59 | 2021-03-03 14:05:54 -0500 | [diff] [blame] | 3376 | if default_platforms and not tc.build_on_all and not integration: |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3377 | if tc.platform_allow: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3378 | a = set(self.default_platforms) |
Anas Nashif | dca317c | 2020-08-26 11:28:25 -0400 | [diff] [blame] | 3379 | b = set(tc.platform_allow) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3380 | c = a.intersection(b) |
| 3381 | if c: |
| 3382 | aa = list(filter(lambda tc: tc.platform.name in c, instance_list)) |
| 3383 | self.add_instances(aa) |
| 3384 | else: |
Kumar Gala | d590e0d | 2021-04-02 10:18:01 -0500 | [diff] [blame] | 3385 | self.add_instances(instance_list) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3386 | else: |
| 3387 | instances = list(filter(lambda tc: tc.platform.default, instance_list)) |
| 3388 | self.add_instances(instances) |
Anas Nashif | 4bccc1d | 2021-03-12 18:21:29 -0500 | [diff] [blame] | 3389 | elif integration: |
Anas Nashif | e618a59 | 2021-03-03 14:05:54 -0500 | [diff] [blame] | 3390 | instances = list(filter(lambda item: item.platform.name in tc.integration_platforms, instance_list)) |
| 3391 | self.add_instances(instances) |
| 3392 | |
| 3393 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3394 | |
Anas Nashif | 9eb9c4c | 2020-08-26 15:47:25 -0400 | [diff] [blame] | 3395 | elif emulation_platforms: |
| 3396 | self.add_instances(instance_list) |
| 3397 | for instance in list(filter(lambda inst: not inst.platform.simulation != 'na', instance_list)): |
| 3398 | discards[instance] = discards.get(instance, "Not an emulated platform") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3399 | else: |
| 3400 | self.add_instances(instance_list) |
| 3401 | |
| 3402 | for _, case in self.instances.items(): |
Christian Taedcke | 3dbe9f2 | 2020-07-06 16:00:57 +0200 | [diff] [blame] | 3403 | case.create_overlay(case.platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3404 | |
| 3405 | self.discards = discards |
| 3406 | self.selected_platforms = set(p.platform.name for p in self.instances.values()) |
| 3407 | |
Maciej Perkowski | 2253c6e | 2021-10-21 14:05:54 +0200 | [diff] [blame] | 3408 | remove_from_discards = [] # configurations to be removed from discards. |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3409 | for instance in self.discards: |
| 3410 | instance.reason = self.discards[instance] |
Maciej Perkowski | d08ee92 | 2021-03-30 14:12:02 +0200 | [diff] [blame] | 3411 | # If integration mode is on all skips on integration_platforms are treated as errors. |
Maciej Perkowski | d871603 | 2021-07-22 15:41:18 +0200 | [diff] [blame] | 3412 | if self.integration and instance.platform.name in instance.testcase.integration_platforms \ |
| 3413 | and "Quarantine" not in instance.reason: |
Maciej Perkowski | d08ee92 | 2021-03-30 14:12:02 +0200 | [diff] [blame] | 3414 | instance.status = "error" |
| 3415 | instance.reason += " but is one of the integration platforms" |
| 3416 | instance.fill_results_by_status() |
| 3417 | self.instances[instance.name] = instance |
Maciej Perkowski | 2253c6e | 2021-10-21 14:05:54 +0200 | [diff] [blame] | 3418 | # Such configuration has to be removed from discards to make sure it won't get skipped |
| 3419 | remove_from_discards.append(instance) |
Maciej Perkowski | d08ee92 | 2021-03-30 14:12:02 +0200 | [diff] [blame] | 3420 | else: |
| 3421 | instance.status = "skipped" |
| 3422 | instance.fill_results_by_status() |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3423 | |
Maciej Perkowski | b6a69aa | 2021-02-04 16:05:21 +0100 | [diff] [blame] | 3424 | self.filtered_platforms = set(p.platform.name for p in self.instances.values() |
| 3425 | if p.status != "skipped" ) |
| 3426 | |
Maciej Perkowski | 2253c6e | 2021-10-21 14:05:54 +0200 | [diff] [blame] | 3427 | # Remove from discards configururations that must not be discarded (e.g. integration_platforms when --integration was used) |
| 3428 | for instance in remove_from_discards: |
| 3429 | del self.discards[instance] |
| 3430 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3431 | return discards |
| 3432 | |
| 3433 | def add_instances(self, instance_list): |
| 3434 | for instance in instance_list: |
| 3435 | self.instances[instance.name] = instance |
| 3436 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3437 | @staticmethod |
| 3438 | def calc_one_elf_size(instance): |
| 3439 | if instance.status not in ["error", "failed", "skipped"]: |
| 3440 | if instance.platform.type != "native": |
| 3441 | size_calc = instance.calculate_sizes() |
| 3442 | instance.metrics["ram_size"] = size_calc.get_ram_size() |
| 3443 | instance.metrics["rom_size"] = size_calc.get_rom_size() |
| 3444 | instance.metrics["unrecognized"] = size_calc.unrecognized_sections() |
| 3445 | else: |
| 3446 | instance.metrics["ram_size"] = 0 |
| 3447 | instance.metrics["rom_size"] = 0 |
| 3448 | instance.metrics["unrecognized"] = [] |
| 3449 | |
| 3450 | instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0 |
| 3451 | |
| 3452 | def add_tasks_to_queue(self, pipeline, build_only=False, test_only=False): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3453 | for instance in self.instances.values(): |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3454 | if build_only: |
| 3455 | instance.run = False |
| 3456 | |
Andreas Vibeto | 32fbf8e | 2021-07-16 10:01:49 +0200 | [diff] [blame] | 3457 | if instance.status not in ['passed', 'skipped', 'error']: |
| 3458 | logger.debug(f"adding {instance.name}") |
| 3459 | instance.status = None |
| 3460 | if test_only and instance.run: |
| 3461 | pipeline.put({"op": "run", "test": instance}) |
| 3462 | else: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3463 | pipeline.put({"op": "cmake", "test": instance}) |
Andreas Vibeto | 32fbf8e | 2021-07-16 10:01:49 +0200 | [diff] [blame] | 3464 | # If the instance got 'error' status before, proceed to the report stage |
| 3465 | if instance.status == "error": |
| 3466 | pipeline.put({"op": "report", "test": instance}) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3467 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3468 | def pipeline_mgr(self, pipeline, done_queue, lock, results): |
| 3469 | while True: |
| 3470 | try: |
| 3471 | task = pipeline.get_nowait() |
| 3472 | except queue.Empty: |
| 3473 | break |
| 3474 | else: |
| 3475 | test = task['test'] |
| 3476 | pb = ProjectBuilder(self, |
| 3477 | test, |
| 3478 | lsan=self.enable_lsan, |
| 3479 | asan=self.enable_asan, |
| 3480 | ubsan=self.enable_ubsan, |
| 3481 | coverage=self.enable_coverage, |
| 3482 | extra_args=self.extra_args, |
| 3483 | device_testing=self.device_testing, |
| 3484 | cmake_only=self.cmake_only, |
| 3485 | cleanup=self.cleanup, |
| 3486 | valgrind=self.enable_valgrind, |
| 3487 | inline_logs=self.inline_logs, |
| 3488 | generator=self.generator, |
| 3489 | generator_cmd=self.generator_cmd, |
| 3490 | verbose=self.verbose, |
Anas Nashif | f68146f | 2020-11-28 11:07:06 -0500 | [diff] [blame] | 3491 | warnings_as_errors=self.warnings_as_errors, |
| 3492 | overflow_as_errors=self.overflow_as_errors |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3493 | ) |
| 3494 | pb.process(pipeline, done_queue, task, lock, results) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3495 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3496 | return True |
Anas Nashif | dc43c29 | 2020-07-09 09:46:45 -0400 | [diff] [blame] | 3497 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3498 | def execute(self, pipeline, done, results): |
| 3499 | lock = Lock() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3500 | logger.info("Adding tasks to the queue...") |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3501 | self.add_tasks_to_queue(pipeline, self.build_only, self.test_only) |
| 3502 | logger.info("Added initial list of jobs to queue") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3503 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3504 | processes = [] |
| 3505 | for job in range(self.jobs): |
| 3506 | logger.debug(f"Launch process {job}") |
| 3507 | p = Process(target=self.pipeline_mgr, args=(pipeline, done, lock, results, )) |
| 3508 | processes.append(p) |
| 3509 | p.start() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3510 | |
Anas Nashif | 2d48917 | 2020-12-09 09:06:57 -0500 | [diff] [blame] | 3511 | try: |
| 3512 | for p in processes: |
| 3513 | p.join() |
| 3514 | except KeyboardInterrupt: |
| 3515 | logger.info("Execution interrupted") |
| 3516 | for p in processes: |
| 3517 | p.terminate() |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3518 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3519 | # FIXME: This needs to move out. |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3520 | if self.enable_size_report and not self.cmake_only: |
| 3521 | # Parallelize size calculation |
| 3522 | executor = concurrent.futures.ThreadPoolExecutor(self.jobs) |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3523 | futures = [executor.submit(self.calc_one_elf_size, instance) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3524 | for instance in self.instances.values()] |
| 3525 | concurrent.futures.wait(futures) |
| 3526 | else: |
| 3527 | for instance in self.instances.values(): |
| 3528 | instance.metrics["ram_size"] = 0 |
| 3529 | instance.metrics["rom_size"] = 0 |
| 3530 | instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0 |
| 3531 | instance.metrics["unrecognized"] = [] |
| 3532 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 3533 | return results |
| 3534 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3535 | def discard_report(self, filename): |
| 3536 | |
| 3537 | try: |
Aastha Grover | dcbd915 | 2020-06-16 10:19:51 -0700 | [diff] [blame] | 3538 | if not self.discards: |
Anas Nashif | 4594370 | 2020-12-11 17:55:15 -0500 | [diff] [blame] | 3539 | raise TwisterRuntimeError("apply_filters() hasn't been run!") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3540 | except Exception as e: |
| 3541 | logger.error(str(e)) |
| 3542 | sys.exit(2) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3543 | with open(filename, "wt") as csvfile: |
| 3544 | fieldnames = ["test", "arch", "platform", "reason"] |
| 3545 | cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) |
| 3546 | cw.writeheader() |
| 3547 | for instance, reason in sorted(self.discards.items()): |
| 3548 | rowdict = {"test": instance.testcase.name, |
| 3549 | "arch": instance.platform.arch, |
| 3550 | "platform": instance.platform.name, |
| 3551 | "reason": reason} |
| 3552 | cw.writerow(rowdict) |
| 3553 | |
Anas Nashif | 6915adf | 2020-04-22 09:39:42 -0400 | [diff] [blame] | 3554 | def target_report(self, outdir, suffix, append=False): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3555 | platforms = {inst.platform.name for _, inst in self.instances.items()} |
| 3556 | for platform in platforms: |
Anas Nashif | 6915adf | 2020-04-22 09:39:42 -0400 | [diff] [blame] | 3557 | if suffix: |
| 3558 | filename = os.path.join(outdir,"{}_{}.xml".format(platform, suffix)) |
| 3559 | else: |
| 3560 | filename = os.path.join(outdir,"{}.xml".format(platform)) |
Maciej Perkowski | 060e00d | 2020-10-13 18:59:37 +0200 | [diff] [blame] | 3561 | self.xunit_report(filename, platform, full_report=True, |
| 3562 | append=append, version=self.version) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3563 | |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3564 | |
| 3565 | @staticmethod |
| 3566 | def process_log(log_file): |
| 3567 | filtered_string = "" |
| 3568 | if os.path.exists(log_file): |
| 3569 | with open(log_file, "rb") as f: |
| 3570 | log = f.read().decode("utf-8") |
| 3571 | filtered_string = ''.join(filter(lambda x: x in string.printable, log)) |
| 3572 | |
| 3573 | return filtered_string |
| 3574 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3575 | |
Maciej Perkowski | 725d19b | 2020-10-13 14:15:12 +0200 | [diff] [blame] | 3576 | def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA"): |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3577 | total = 0 |
Anas Nashif | b517b1f | 2020-12-11 07:38:49 -0500 | [diff] [blame] | 3578 | fails = passes = errors = skips = 0 |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3579 | if platform: |
| 3580 | selected = [platform] |
Anas Nashif | b517b1f | 2020-12-11 07:38:49 -0500 | [diff] [blame] | 3581 | logger.info(f"Writing target report for {platform}...") |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3582 | else: |
Anas Nashif | b517b1f | 2020-12-11 07:38:49 -0500 | [diff] [blame] | 3583 | logger.info(f"Writing xunit report {filename}...") |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3584 | selected = self.selected_platforms |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3585 | |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3586 | if os.path.exists(filename) and append: |
| 3587 | tree = ET.parse(filename) |
| 3588 | eleTestsuites = tree.getroot() |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3589 | else: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3590 | eleTestsuites = ET.Element('testsuites') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3591 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3592 | for p in selected: |
| 3593 | inst = self.get_platform_instances(p) |
| 3594 | fails = 0 |
| 3595 | passes = 0 |
| 3596 | errors = 0 |
| 3597 | skips = 0 |
| 3598 | duration = 0 |
| 3599 | |
| 3600 | for _, instance in inst.items(): |
| 3601 | handler_time = instance.metrics.get('handler_time', 0) |
| 3602 | duration += handler_time |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3603 | if full_report and instance.run: |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3604 | for k in instance.results.keys(): |
| 3605 | if instance.results[k] == 'PASS': |
| 3606 | passes += 1 |
| 3607 | elif instance.results[k] == 'BLOCK': |
| 3608 | errors += 1 |
Anas Nashif | be1025a | 2020-10-31 08:04:48 -0400 | [diff] [blame] | 3609 | elif instance.results[k] == 'SKIP' or instance.status in ['skipped']: |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3610 | skips += 1 |
| 3611 | else: |
| 3612 | fails += 1 |
| 3613 | else: |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 3614 | if instance.status in ["error", "failed", "timeout", "flash_error"]: |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3615 | if instance.reason in ['build_error', 'handler_crash']: |
| 3616 | errors += 1 |
| 3617 | else: |
| 3618 | fails += 1 |
| 3619 | elif instance.status == 'skipped': |
| 3620 | skips += 1 |
Anas Nashif | 9e1be4c | 2020-07-23 08:20:49 -0400 | [diff] [blame] | 3621 | elif instance.status == 'passed': |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3622 | passes += 1 |
Anas Nashif | 9e1be4c | 2020-07-23 08:20:49 -0400 | [diff] [blame] | 3623 | else: |
Anas Nashif | 696476e | 2020-11-24 08:42:46 -0500 | [diff] [blame] | 3624 | if instance.status: |
| 3625 | logger.error(f"{instance.name}: Unknown status {instance.status}") |
| 3626 | else: |
| 3627 | logger.error(f"{instance.name}: No status") |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3628 | |
| 3629 | total = (errors + passes + fails + skips) |
| 3630 | # do not produce a report if no tests were actually run (only built) |
| 3631 | if total == 0: |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3632 | continue |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3633 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3634 | run = p |
| 3635 | eleTestsuite = None |
| 3636 | |
| 3637 | # When we re-run the tests, we re-use the results and update only with |
| 3638 | # the newly run tests. |
| 3639 | if os.path.exists(filename) and append: |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3640 | ts = eleTestsuites.findall(f'testsuite/[@name="{p}"]') |
| 3641 | if ts: |
| 3642 | eleTestsuite = ts[0] |
| 3643 | eleTestsuite.attrib['failures'] = "%d" % fails |
| 3644 | eleTestsuite.attrib['errors'] = "%d" % errors |
Christian Taedcke | b2be804 | 2020-08-12 14:21:13 +0200 | [diff] [blame] | 3645 | eleTestsuite.attrib['skipped'] = "%d" % skips |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3646 | else: |
| 3647 | logger.info(f"Did not find any existing results for {p}") |
| 3648 | eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite', |
| 3649 | name=run, time="%f" % duration, |
| 3650 | tests="%d" % (total), |
| 3651 | failures="%d" % fails, |
Christian Taedcke | b2be804 | 2020-08-12 14:21:13 +0200 | [diff] [blame] | 3652 | errors="%d" % (errors), skipped="%s" % (skips)) |
Maciej Perkowski | 725d19b | 2020-10-13 14:15:12 +0200 | [diff] [blame] | 3653 | eleTSPropetries = ET.SubElement(eleTestsuite, 'properties') |
| 3654 | # Multiple 'property' can be added to 'properties' |
| 3655 | # differing by name and value |
Maciej Perkowski | 060e00d | 2020-10-13 18:59:37 +0200 | [diff] [blame] | 3656 | ET.SubElement(eleTSPropetries, 'property', name="version", value=version) |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3657 | |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3658 | else: |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3659 | eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite', |
| 3660 | name=run, time="%f" % duration, |
| 3661 | tests="%d" % (total), |
| 3662 | failures="%d" % fails, |
Christian Taedcke | b2be804 | 2020-08-12 14:21:13 +0200 | [diff] [blame] | 3663 | errors="%d" % (errors), skipped="%s" % (skips)) |
Maciej Perkowski | 725d19b | 2020-10-13 14:15:12 +0200 | [diff] [blame] | 3664 | eleTSPropetries = ET.SubElement(eleTestsuite, 'properties') |
| 3665 | # Multiple 'property' can be added to 'properties' |
| 3666 | # differing by name and value |
Maciej Perkowski | 060e00d | 2020-10-13 18:59:37 +0200 | [diff] [blame] | 3667 | ET.SubElement(eleTSPropetries, 'property', name="version", value=version) |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3668 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3669 | for _, instance in inst.items(): |
| 3670 | if full_report: |
| 3671 | tname = os.path.basename(instance.testcase.name) |
| 3672 | else: |
| 3673 | tname = instance.testcase.id |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3674 | handler_time = instance.metrics.get('handler_time', 0) |
| 3675 | |
| 3676 | if full_report: |
| 3677 | for k in instance.results.keys(): |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3678 | # remove testcases that are being re-run from exiting reports |
| 3679 | for tc in eleTestsuite.findall(f'testcase/[@name="{k}"]'): |
| 3680 | eleTestsuite.remove(tc) |
| 3681 | |
| 3682 | classname = ".".join(tname.split(".")[:2]) |
| 3683 | eleTestcase = ET.SubElement( |
| 3684 | eleTestsuite, 'testcase', |
| 3685 | classname=classname, |
| 3686 | name="%s" % (k), time="%f" % handler_time) |
Anas Nashif | 9e1be4c | 2020-07-23 08:20:49 -0400 | [diff] [blame] | 3687 | if instance.results[k] in ['FAIL', 'BLOCK'] or \ |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3688 | (not instance.run and instance.status in ["error", "failed", "timeout"]): |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3689 | if instance.results[k] == 'FAIL': |
| 3690 | el = ET.SubElement( |
| 3691 | eleTestcase, |
| 3692 | 'failure', |
| 3693 | type="failure", |
| 3694 | message="failed") |
| 3695 | else: |
| 3696 | el = ET.SubElement( |
| 3697 | eleTestcase, |
| 3698 | 'error', |
| 3699 | type="failure", |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 3700 | message=instance.reason) |
Jingru Wang | 99b0720 | 2021-01-29 12:42:32 +0800 | [diff] [blame] | 3701 | log_root = os.path.join(self.outdir, instance.platform.name, instance.testcase.name) |
| 3702 | log_file = os.path.join(log_root, "handler.log") |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3703 | el.text = self.process_log(log_file) |
| 3704 | |
Anas Nashif | 9e1be4c | 2020-07-23 08:20:49 -0400 | [diff] [blame] | 3705 | elif instance.results[k] == 'PASS' \ |
Anas Nashif | 405f1b6 | 2020-07-27 12:27:13 -0400 | [diff] [blame] | 3706 | or (not instance.run and instance.status in ["passed"]): |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3707 | pass |
Anas Nashif | be1025a | 2020-10-31 08:04:48 -0400 | [diff] [blame] | 3708 | elif instance.results[k] == 'SKIP' or (instance.status in ["skipped"]): |
Maciej Perkowski | e3ff4cf | 2020-07-17 11:13:50 +0200 | [diff] [blame] | 3709 | el = ET.SubElement(eleTestcase, 'skipped', type="skipped", message=instance.reason) |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3710 | else: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3711 | el = ET.SubElement( |
| 3712 | eleTestcase, |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3713 | 'error', |
| 3714 | type="error", |
| 3715 | message=f"{instance.reason}") |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3716 | else: |
| 3717 | if platform: |
| 3718 | classname = ".".join(instance.testcase.name.split(".")[:2]) |
| 3719 | else: |
| 3720 | classname = p + ":" + ".".join(instance.testcase.name.split(".")[:2]) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3721 | |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3722 | # remove testcases that are being re-run from exiting reports |
Sebastian Wezel | 148dbdc | 2021-02-22 13:04:56 +0100 | [diff] [blame] | 3723 | for tc in eleTestsuite.findall(f'testcase/[@classname="{classname}"][@name="{instance.testcase.name}"]'): |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3724 | eleTestsuite.remove(tc) |
| 3725 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3726 | eleTestcase = ET.SubElement(eleTestsuite, 'testcase', |
| 3727 | classname=classname, |
| 3728 | name="%s" % (instance.testcase.name), |
| 3729 | time="%f" % handler_time) |
Anas Nashif | 9e1be4c | 2020-07-23 08:20:49 -0400 | [diff] [blame] | 3730 | |
Maciej Perkowski | f050a99 | 2021-02-24 13:43:05 +0100 | [diff] [blame] | 3731 | if instance.status in ["error", "failed", "timeout", "flash_error"]: |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3732 | failure = ET.SubElement( |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3733 | eleTestcase, |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3734 | 'failure', |
| 3735 | type="failure", |
Maciej Perkowski | b2fa99c | 2020-05-21 14:45:29 +0200 | [diff] [blame] | 3736 | message=instance.reason) |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3737 | |
Jingru Wang | 99b0720 | 2021-01-29 12:42:32 +0800 | [diff] [blame] | 3738 | log_root = ("%s/%s/%s" % (self.outdir, instance.platform.name, instance.testcase.name)) |
| 3739 | bl = os.path.join(log_root, "build.log") |
| 3740 | hl = os.path.join(log_root, "handler.log") |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3741 | log_file = bl |
| 3742 | if instance.reason != 'Build error': |
| 3743 | if os.path.exists(hl): |
| 3744 | log_file = hl |
| 3745 | else: |
| 3746 | log_file = bl |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3747 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3748 | failure.text = self.process_log(log_file) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3749 | |
Anas Nashif | a53c813 | 2020-05-05 09:32:46 -0400 | [diff] [blame] | 3750 | elif instance.status == "skipped": |
| 3751 | ET.SubElement(eleTestcase, 'skipped', type="skipped", message="Skipped") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3752 | |
| 3753 | result = ET.tostring(eleTestsuites) |
| 3754 | with open(filename, 'wb') as report: |
| 3755 | report.write(result) |
| 3756 | |
Anas Nashif | 1c2f127 | 2020-07-23 09:56:01 -0400 | [diff] [blame] | 3757 | return fails, passes, errors, skips |
Anas Nashif | 9041550 | 2020-04-11 22:15:04 -0400 | [diff] [blame] | 3758 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3759 | def csv_report(self, filename): |
| 3760 | with open(filename, "wt") as csvfile: |
| 3761 | fieldnames = ["test", "arch", "platform", "status", |
| 3762 | "extra_args", "handler", "handler_time", "ram_size", |
| 3763 | "rom_size"] |
| 3764 | cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) |
| 3765 | cw.writeheader() |
| 3766 | for instance in self.instances.values(): |
| 3767 | rowdict = {"test": instance.testcase.name, |
| 3768 | "arch": instance.platform.arch, |
| 3769 | "platform": instance.platform.name, |
| 3770 | "extra_args": " ".join(instance.testcase.extra_args), |
| 3771 | "handler": instance.platform.simulation} |
| 3772 | |
| 3773 | rowdict["status"] = instance.status |
Anas Nashif | f04461e | 2020-06-29 10:07:02 -0400 | [diff] [blame] | 3774 | if instance.status not in ["error", "failed", "timeout"]: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3775 | if instance.handler: |
| 3776 | rowdict["handler_time"] = instance.metrics.get("handler_time", 0) |
| 3777 | ram_size = instance.metrics.get("ram_size", 0) |
| 3778 | rom_size = instance.metrics.get("rom_size", 0) |
| 3779 | rowdict["ram_size"] = ram_size |
| 3780 | rowdict["rom_size"] = rom_size |
| 3781 | cw.writerow(rowdict) |
| 3782 | |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3783 | def json_report(self, filename, append=False, version="NA"): |
Anas Nashif | b517b1f | 2020-12-11 07:38:49 -0500 | [diff] [blame] | 3784 | logger.info(f"Writing JSON report {filename}") |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3785 | report = {} |
| 3786 | selected = self.selected_platforms |
| 3787 | report["environment"] = {"os": os.name, |
| 3788 | "zephyr_version": version, |
| 3789 | "toolchain": self.get_toolchain() |
| 3790 | } |
| 3791 | json_data = {} |
| 3792 | if os.path.exists(filename) and append: |
| 3793 | with open(filename, 'r') as json_file: |
| 3794 | json_data = json.load(json_file) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3795 | |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3796 | suites = json_data.get("testsuites", []) |
| 3797 | if suites: |
| 3798 | suite = suites[0] |
| 3799 | testcases = suite.get("testcases", []) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3800 | else: |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3801 | suite = {} |
| 3802 | testcases = [] |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3803 | |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3804 | for p in selected: |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3805 | inst = self.get_platform_instances(p) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3806 | for _, instance in inst.items(): |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3807 | testcase = {} |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3808 | handler_log = os.path.join(instance.build_dir, "handler.log") |
| 3809 | build_log = os.path.join(instance.build_dir, "build.log") |
| 3810 | device_log = os.path.join(instance.build_dir, "device.log") |
| 3811 | |
| 3812 | handler_time = instance.metrics.get('handler_time', 0) |
| 3813 | ram_size = instance.metrics.get ("ram_size", 0) |
| 3814 | rom_size = instance.metrics.get("rom_size",0) |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3815 | for k in instance.results.keys(): |
| 3816 | testcases = list(filter(lambda d: not (d.get('testcase') == k and d.get('platform') == p), testcases )) |
| 3817 | testcase = {"testcase": k, |
| 3818 | "arch": instance.platform.arch, |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3819 | "platform": p, |
| 3820 | } |
Maciej Perkowski | 484ef67 | 2021-04-15 11:37:39 +0200 | [diff] [blame] | 3821 | if ram_size: |
| 3822 | testcase["ram_size"] = ram_size |
| 3823 | if rom_size: |
| 3824 | testcase["rom_size"] = rom_size |
| 3825 | |
Anas Nashif | 55c3dde | 2021-09-16 08:54:19 -0400 | [diff] [blame] | 3826 | if instance.results[k] in ["PASS"] or instance.status == 'passed': |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3827 | testcase["status"] = "passed" |
| 3828 | if instance.handler: |
| 3829 | testcase["execution_time"] = handler_time |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3830 | |
Anas Nashif | 55c3dde | 2021-09-16 08:54:19 -0400 | [diff] [blame] | 3831 | elif instance.results[k] in ['FAIL', 'BLOCK'] or instance.status in ["error", "failed", "timeout", "flash_error"]: |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3832 | testcase["status"] = "failed" |
| 3833 | testcase["reason"] = instance.reason |
| 3834 | testcase["execution_time"] = handler_time |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3835 | if os.path.exists(handler_log): |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3836 | testcase["test_output"] = self.process_log(handler_log) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3837 | elif os.path.exists(device_log): |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3838 | testcase["device_log"] = self.process_log(device_log) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3839 | else: |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3840 | testcase["build_log"] = self.process_log(build_log) |
Anas Nashif | 55c3dde | 2021-09-16 08:54:19 -0400 | [diff] [blame] | 3841 | elif instance.status == 'skipped': |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3842 | testcase["status"] = "skipped" |
| 3843 | testcase["reason"] = instance.reason |
| 3844 | testcases.append(testcase) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3845 | |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3846 | suites = [ {"testcases": testcases} ] |
| 3847 | report["testsuites"] = suites |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3848 | |
| 3849 | with open(filename, "wt") as json_file: |
Anas Nashif | cd1dccf | 2020-12-10 15:07:52 -0500 | [diff] [blame] | 3850 | json.dump(report, json_file, indent=4, separators=(',',':')) |
Spoorthy Priya Yerabolu | f8f220a | 2020-09-23 06:28:50 -0700 | [diff] [blame] | 3851 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3852 | def get_testcase(self, identifier): |
| 3853 | results = [] |
| 3854 | for _, tc in self.testcases.items(): |
| 3855 | for case in tc.cases: |
| 3856 | if case == identifier: |
| 3857 | results.append(tc) |
| 3858 | return results |
| 3859 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3860 | class CoverageTool: |
| 3861 | """ Base class for every supported coverage tool |
| 3862 | """ |
| 3863 | |
| 3864 | def __init__(self): |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3865 | self.gcov_tool = None |
| 3866 | self.base_dir = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3867 | |
| 3868 | @staticmethod |
| 3869 | def factory(tool): |
| 3870 | if tool == 'lcov': |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3871 | t = Lcov() |
| 3872 | elif tool == 'gcovr': |
Marcin Niestroj | 2652dc7 | 2020-08-10 22:27:03 +0200 | [diff] [blame] | 3873 | t = Gcovr() |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3874 | else: |
| 3875 | logger.error("Unsupported coverage tool specified: {}".format(tool)) |
| 3876 | return None |
| 3877 | |
Anas Nashif | 211ef41 | 2021-01-12 07:15:59 -0500 | [diff] [blame] | 3878 | logger.debug(f"Select {tool} as the coverage tool...") |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3879 | return t |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3880 | |
| 3881 | @staticmethod |
Kentaro Sugimoto | e8ab987 | 2021-07-01 16:44:52 +0900 | [diff] [blame] | 3882 | def retrieve_gcov_data(input_file): |
| 3883 | logger.debug("Working on %s" % input_file) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3884 | extracted_coverage_info = {} |
| 3885 | capture_data = False |
| 3886 | capture_complete = False |
Kentaro Sugimoto | e8ab987 | 2021-07-01 16:44:52 +0900 | [diff] [blame] | 3887 | with open(input_file, 'r') as fp: |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3888 | for line in fp.readlines(): |
| 3889 | if re.search("GCOV_COVERAGE_DUMP_START", line): |
| 3890 | capture_data = True |
| 3891 | continue |
| 3892 | if re.search("GCOV_COVERAGE_DUMP_END", line): |
| 3893 | capture_complete = True |
| 3894 | break |
| 3895 | # Loop until the coverage data is found. |
| 3896 | if not capture_data: |
| 3897 | continue |
| 3898 | if line.startswith("*"): |
| 3899 | sp = line.split("<") |
| 3900 | if len(sp) > 1: |
| 3901 | # Remove the leading delimiter "*" |
| 3902 | file_name = sp[0][1:] |
| 3903 | # Remove the trailing new line char |
| 3904 | hex_dump = sp[1][:-1] |
| 3905 | else: |
| 3906 | continue |
| 3907 | else: |
| 3908 | continue |
| 3909 | extracted_coverage_info.update({file_name: hex_dump}) |
| 3910 | if not capture_data: |
| 3911 | capture_complete = True |
| 3912 | return {'complete': capture_complete, 'data': extracted_coverage_info} |
| 3913 | |
| 3914 | @staticmethod |
| 3915 | def create_gcda_files(extracted_coverage_info): |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3916 | logger.debug("Generating gcda files") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3917 | for filename, hexdump_val in extracted_coverage_info.items(): |
| 3918 | # if kobject_hash is given for coverage gcovr fails |
| 3919 | # hence skipping it problem only in gcovr v4.1 |
| 3920 | if "kobject_hash" in filename: |
| 3921 | filename = (filename[:-4]) + "gcno" |
| 3922 | try: |
| 3923 | os.remove(filename) |
| 3924 | except Exception: |
| 3925 | pass |
| 3926 | continue |
| 3927 | |
| 3928 | with open(filename, 'wb') as fp: |
| 3929 | fp.write(bytes.fromhex(hexdump_val)) |
| 3930 | |
| 3931 | def generate(self, outdir): |
| 3932 | for filename in glob.glob("%s/**/handler.log" % outdir, recursive=True): |
| 3933 | gcov_data = self.__class__.retrieve_gcov_data(filename) |
| 3934 | capture_complete = gcov_data['complete'] |
| 3935 | extracted_coverage_info = gcov_data['data'] |
| 3936 | if capture_complete: |
| 3937 | self.__class__.create_gcda_files(extracted_coverage_info) |
| 3938 | logger.debug("Gcov data captured: {}".format(filename)) |
| 3939 | else: |
| 3940 | logger.error("Gcov data capture incomplete: {}".format(filename)) |
| 3941 | |
| 3942 | with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog: |
| 3943 | ret = self._generate(outdir, coveragelog) |
| 3944 | if ret == 0: |
| 3945 | logger.info("HTML report generated: {}".format( |
| 3946 | os.path.join(outdir, "coverage", "index.html"))) |
| 3947 | |
| 3948 | |
| 3949 | class Lcov(CoverageTool): |
| 3950 | |
| 3951 | def __init__(self): |
| 3952 | super().__init__() |
| 3953 | self.ignores = [] |
| 3954 | |
| 3955 | def add_ignore_file(self, pattern): |
| 3956 | self.ignores.append('*' + pattern + '*') |
| 3957 | |
| 3958 | def add_ignore_directory(self, pattern): |
Marcin Niestroj | fc67409 | 2020-08-10 23:22:11 +0200 | [diff] [blame] | 3959 | self.ignores.append('*/' + pattern + '/*') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3960 | |
| 3961 | def _generate(self, outdir, coveragelog): |
| 3962 | coveragefile = os.path.join(outdir, "coverage.info") |
| 3963 | ztestfile = os.path.join(outdir, "ztest.info") |
Anas Nashif | 211ef41 | 2021-01-12 07:15:59 -0500 | [diff] [blame] | 3964 | cmd = ["lcov", "--gcov-tool", self.gcov_tool, |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3965 | "--capture", "--directory", outdir, |
| 3966 | "--rc", "lcov_branch_coverage=1", |
Anas Nashif | 211ef41 | 2021-01-12 07:15:59 -0500 | [diff] [blame] | 3967 | "--output-file", coveragefile] |
| 3968 | cmd_str = " ".join(cmd) |
| 3969 | logger.debug(f"Running {cmd_str}...") |
| 3970 | subprocess.call(cmd, stdout=coveragelog) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3971 | # We want to remove tests/* and tests/ztest/test/* but save tests/ztest |
| 3972 | subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--extract", |
| 3973 | coveragefile, |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3974 | os.path.join(self.base_dir, "tests", "ztest", "*"), |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3975 | "--output-file", ztestfile, |
| 3976 | "--rc", "lcov_branch_coverage=1"], stdout=coveragelog) |
| 3977 | |
| 3978 | if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0: |
| 3979 | subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--remove", |
| 3980 | ztestfile, |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 3981 | os.path.join(self.base_dir, "tests/ztest/test/*"), |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 3982 | "--output-file", ztestfile, |
| 3983 | "--rc", "lcov_branch_coverage=1"], |
| 3984 | stdout=coveragelog) |
| 3985 | files = [coveragefile, ztestfile] |
| 3986 | else: |
| 3987 | files = [coveragefile] |
| 3988 | |
| 3989 | for i in self.ignores: |
| 3990 | subprocess.call( |
| 3991 | ["lcov", "--gcov-tool", self.gcov_tool, "--remove", |
| 3992 | coveragefile, i, "--output-file", |
| 3993 | coveragefile, "--rc", "lcov_branch_coverage=1"], |
| 3994 | stdout=coveragelog) |
| 3995 | |
| 3996 | # The --ignore-errors source option is added to avoid it exiting due to |
| 3997 | # samples/application_development/external_lib/ |
| 3998 | return subprocess.call(["genhtml", "--legend", "--branch-coverage", |
| 3999 | "--ignore-errors", "source", |
| 4000 | "-output-directory", |
| 4001 | os.path.join(outdir, "coverage")] + files, |
| 4002 | stdout=coveragelog) |
| 4003 | |
| 4004 | |
| 4005 | class Gcovr(CoverageTool): |
| 4006 | |
| 4007 | def __init__(self): |
| 4008 | super().__init__() |
| 4009 | self.ignores = [] |
| 4010 | |
| 4011 | def add_ignore_file(self, pattern): |
| 4012 | self.ignores.append('.*' + pattern + '.*') |
| 4013 | |
| 4014 | def add_ignore_directory(self, pattern): |
Marcin Niestroj | fc67409 | 2020-08-10 23:22:11 +0200 | [diff] [blame] | 4015 | self.ignores.append(".*/" + pattern + '/.*') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4016 | |
| 4017 | @staticmethod |
| 4018 | def _interleave_list(prefix, list): |
| 4019 | tuple_list = [(prefix, item) for item in list] |
| 4020 | return [item for sublist in tuple_list for item in sublist] |
| 4021 | |
| 4022 | def _generate(self, outdir, coveragelog): |
| 4023 | coveragefile = os.path.join(outdir, "coverage.json") |
| 4024 | ztestfile = os.path.join(outdir, "ztest.json") |
| 4025 | |
| 4026 | excludes = Gcovr._interleave_list("-e", self.ignores) |
| 4027 | |
| 4028 | # We want to remove tests/* and tests/ztest/test/* but save tests/ztest |
Anas Nashif | 211ef41 | 2021-01-12 07:15:59 -0500 | [diff] [blame] | 4029 | cmd = ["gcovr", "-r", self.base_dir, "--gcov-executable", |
| 4030 | self.gcov_tool, "-e", "tests/*"] + excludes + ["--json", "-o", |
| 4031 | coveragefile, outdir] |
| 4032 | cmd_str = " ".join(cmd) |
| 4033 | logger.debug(f"Running {cmd_str}...") |
| 4034 | subprocess.call(cmd, stdout=coveragelog) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4035 | |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 4036 | subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4037 | self.gcov_tool, "-f", "tests/ztest", "-e", |
| 4038 | "tests/ztest/test/*", "--json", "-o", ztestfile, |
| 4039 | outdir], stdout=coveragelog) |
| 4040 | |
| 4041 | if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0: |
| 4042 | files = [coveragefile, ztestfile] |
| 4043 | else: |
| 4044 | files = [coveragefile] |
| 4045 | |
| 4046 | subdir = os.path.join(outdir, "coverage") |
| 4047 | os.makedirs(subdir, exist_ok=True) |
| 4048 | |
| 4049 | tracefiles = self._interleave_list("--add-tracefile", files) |
| 4050 | |
Anas Nashif | f6462a3 | 2020-03-29 19:02:51 -0400 | [diff] [blame] | 4051 | return subprocess.call(["gcovr", "-r", self.base_dir, "--html", |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4052 | "--html-details"] + tracefiles + |
| 4053 | ["-o", os.path.join(subdir, "index.html")], |
| 4054 | stdout=coveragelog) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4055 | |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 4056 | |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4057 | class DUT(object): |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4058 | def __init__(self, |
| 4059 | id=None, |
| 4060 | serial=None, |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 4061 | serial_baud=None, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4062 | platform=None, |
| 4063 | product=None, |
| 4064 | serial_pty=None, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4065 | connected=False, |
| 4066 | pre_script=None, |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4067 | post_script=None, |
| 4068 | post_flash_script=None, |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4069 | runner=None): |
| 4070 | |
| 4071 | self.serial = serial |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 4072 | self.serial_baud = 115200 |
| 4073 | if serial_baud: |
| 4074 | self.serial_baud = serial_baud |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4075 | self.platform = platform |
| 4076 | self.serial_pty = serial_pty |
| 4077 | self._counter = Value("i", 0) |
| 4078 | self._available = Value("i", 1) |
| 4079 | self.connected = connected |
| 4080 | self.pre_script = pre_script |
| 4081 | self.id = id |
| 4082 | self.product = product |
| 4083 | self.runner = runner |
| 4084 | self.fixtures = [] |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4085 | self.post_flash_script = post_flash_script |
| 4086 | self.post_script = post_script |
| 4087 | self.pre_script = pre_script |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4088 | self.probe_id = None |
| 4089 | self.notes = None |
Andy Ross | 098fce3 | 2021-03-02 05:13:07 -0800 | [diff] [blame] | 4090 | self.lock = Lock() |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4091 | self.match = False |
| 4092 | |
| 4093 | |
| 4094 | @property |
| 4095 | def available(self): |
| 4096 | with self._available.get_lock(): |
| 4097 | return self._available.value |
| 4098 | |
| 4099 | @available.setter |
| 4100 | def available(self, value): |
| 4101 | with self._available.get_lock(): |
| 4102 | self._available.value = value |
| 4103 | |
| 4104 | @property |
| 4105 | def counter(self): |
| 4106 | with self._counter.get_lock(): |
| 4107 | return self._counter.value |
| 4108 | |
| 4109 | @counter.setter |
| 4110 | def counter(self, value): |
| 4111 | with self._counter.get_lock(): |
| 4112 | self._counter.value = value |
| 4113 | |
Anas Nashif | 4fa82c4 | 2020-12-15 10:08:28 -0500 | [diff] [blame] | 4114 | def to_dict(self): |
| 4115 | d = {} |
| 4116 | exclude = ['_available', '_counter', 'match'] |
| 4117 | v = vars(self) |
| 4118 | for k in v.keys(): |
| 4119 | if k not in exclude and v[k]: |
| 4120 | d[k] = v[k] |
| 4121 | return d |
| 4122 | |
| 4123 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4124 | def __repr__(self): |
| 4125 | return f"<{self.platform} ({self.product}) on {self.serial}>" |
| 4126 | |
| 4127 | class HardwareMap: |
Anas Nashif | c24bf6f | 2020-12-07 11:18:07 -0500 | [diff] [blame] | 4128 | schema_path = os.path.join(ZEPHYR_BASE, "scripts", "schemas", "twister", "hwmap-schema.yaml") |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4129 | |
| 4130 | manufacturer = [ |
| 4131 | 'ARM', |
| 4132 | 'SEGGER', |
| 4133 | 'MBED', |
| 4134 | 'STMicroelectronics', |
| 4135 | 'Atmel Corp.', |
| 4136 | 'Texas Instruments', |
| 4137 | 'Silicon Labs', |
| 4138 | 'NXP Semiconductors', |
| 4139 | 'Microchip Technology Inc.', |
| 4140 | 'FTDI', |
| 4141 | 'Digilent' |
| 4142 | ] |
| 4143 | |
| 4144 | runner_mapping = { |
| 4145 | 'pyocd': [ |
| 4146 | 'DAPLink CMSIS-DAP', |
| 4147 | 'MBED CMSIS-DAP' |
| 4148 | ], |
| 4149 | 'jlink': [ |
| 4150 | 'J-Link', |
| 4151 | 'J-Link OB' |
| 4152 | ], |
| 4153 | 'openocd': [ |
Erwan Gouriou | 2339fa0 | 2020-07-07 17:15:22 +0200 | [diff] [blame] | 4154 | 'STM32 STLink', '^XDS110.*', 'STLINK-V3' |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4155 | ], |
| 4156 | 'dediprog': [ |
| 4157 | 'TTL232R-3V3', |
| 4158 | 'MCP2200 USB Serial Port Emulator' |
| 4159 | ] |
| 4160 | } |
| 4161 | |
| 4162 | def __init__(self): |
| 4163 | self.detected = [] |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4164 | self.duts = [] |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4165 | |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 4166 | def add_device(self, serial, platform, pre_script, is_pty, baud=None): |
| 4167 | device = DUT(platform=platform, connected=True, pre_script=pre_script, serial_baud=baud) |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 4168 | |
| 4169 | if is_pty: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4170 | device.serial_pty = serial |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 4171 | else: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4172 | device.serial = serial |
Andrei Emeltchenko | d8b845b | 2020-03-31 13:22:30 +0300 | [diff] [blame] | 4173 | |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4174 | self.duts.append(device) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4175 | |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4176 | def load(self, map_file): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4177 | hwm_schema = scl.yaml_load(self.schema_path) |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4178 | duts = scl.yaml_load_verify(map_file, hwm_schema) |
| 4179 | for dut in duts: |
| 4180 | pre_script = dut.get('pre_script') |
| 4181 | post_script = dut.get('post_script') |
| 4182 | post_flash_script = dut.get('post_flash_script') |
| 4183 | platform = dut.get('platform') |
| 4184 | id = dut.get('id') |
| 4185 | runner = dut.get('runner') |
| 4186 | serial = dut.get('serial') |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 4187 | baud = dut.get('baud', None) |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4188 | product = dut.get('product') |
Anas Nashif | ce954d4 | 2021-02-04 09:31:07 -0500 | [diff] [blame] | 4189 | fixtures = dut.get('fixtures', []) |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4190 | new_dut = DUT(platform=platform, |
| 4191 | product=product, |
| 4192 | runner=runner, |
| 4193 | id=id, |
| 4194 | serial=serial, |
Dennis Ruffer | c714c78 | 2021-10-05 15:34:54 -0700 | [diff] [blame] | 4195 | serial_baud=baud, |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4196 | connected=serial is not None, |
| 4197 | pre_script=pre_script, |
| 4198 | post_script=post_script, |
| 4199 | post_flash_script=post_flash_script) |
Anas Nashif | ce954d4 | 2021-02-04 09:31:07 -0500 | [diff] [blame] | 4200 | new_dut.fixtures = fixtures |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4201 | new_dut.counter = 0 |
| 4202 | self.duts.append(new_dut) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4203 | |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4204 | def scan(self, persistent=False): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4205 | from serial.tools import list_ports |
| 4206 | |
Martí Bolívar | 07dce82 | 2020-04-13 16:50:51 -0700 | [diff] [blame] | 4207 | if persistent and platform.system() == 'Linux': |
| 4208 | # On Linux, /dev/serial/by-id provides symlinks to |
| 4209 | # '/dev/ttyACMx' nodes using names which are unique as |
| 4210 | # long as manufacturers fill out USB metadata nicely. |
| 4211 | # |
| 4212 | # This creates a map from '/dev/ttyACMx' device nodes |
| 4213 | # to '/dev/serial/by-id/usb-...' symlinks. The symlinks |
| 4214 | # go into the hardware map because they stay the same |
| 4215 | # even when the user unplugs / replugs the device. |
| 4216 | # |
| 4217 | # Some inexpensive USB/serial adapters don't result |
| 4218 | # in unique names here, though, so use of this feature |
| 4219 | # requires explicitly setting persistent=True. |
| 4220 | by_id = Path('/dev/serial/by-id') |
| 4221 | def readlink(link): |
| 4222 | return str((by_id / link).resolve()) |
| 4223 | |
| 4224 | persistent_map = {readlink(link): str(link) |
| 4225 | for link in by_id.iterdir()} |
| 4226 | else: |
| 4227 | persistent_map = {} |
| 4228 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4229 | serial_devices = list_ports.comports() |
| 4230 | logger.info("Scanning connected hardware...") |
| 4231 | for d in serial_devices: |
| 4232 | if d.manufacturer in self.manufacturer: |
| 4233 | |
| 4234 | # TI XDS110 can have multiple serial devices for a single board |
| 4235 | # assume endpoint 0 is the serial, skip all others |
| 4236 | if d.manufacturer == 'Texas Instruments' and not d.location.endswith('0'): |
| 4237 | continue |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4238 | s_dev = DUT(platform="unknown", |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4239 | id=d.serial_number, |
| 4240 | serial=persistent_map.get(d.device, d.device), |
| 4241 | product=d.product, |
Anas Nashif | 0e64b87 | 2021-05-17 09:45:03 -0400 | [diff] [blame] | 4242 | runner='unknown', |
| 4243 | connected=True) |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4244 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4245 | for runner, _ in self.runner_mapping.items(): |
| 4246 | products = self.runner_mapping.get(runner) |
| 4247 | if d.product in products: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4248 | s_dev.runner = runner |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4249 | continue |
| 4250 | # Try regex matching |
| 4251 | for p in products: |
| 4252 | if re.match(p, d.product): |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4253 | s_dev.runner = runner |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4254 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4255 | s_dev.connected = True |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4256 | self.detected.append(s_dev) |
| 4257 | else: |
| 4258 | logger.warning("Unsupported device (%s): %s" % (d.manufacturer, d)) |
| 4259 | |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4260 | def save(self, hwm_file): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4261 | # use existing map |
Anas Nashif | 4fa82c4 | 2020-12-15 10:08:28 -0500 | [diff] [blame] | 4262 | self.detected.sort(key=lambda x: x.serial or '') |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4263 | if os.path.exists(hwm_file): |
| 4264 | with open(hwm_file, 'r') as yaml_file: |
Anas Nashif | ae61b7e | 2020-07-06 11:30:55 -0400 | [diff] [blame] | 4265 | hwm = yaml.load(yaml_file, Loader=SafeLoader) |
Anas Nashif | 4fa82c4 | 2020-12-15 10:08:28 -0500 | [diff] [blame] | 4266 | if hwm: |
| 4267 | hwm.sort(key=lambda x: x['serial'] or '') |
Øyvind Rønningstad | 4813f46 | 2020-07-01 16:49:38 +0200 | [diff] [blame] | 4268 | |
Anas Nashif | 4fa82c4 | 2020-12-15 10:08:28 -0500 | [diff] [blame] | 4269 | # disconnect everything |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4270 | for h in hwm: |
Anas Nashif | 4fa82c4 | 2020-12-15 10:08:28 -0500 | [diff] [blame] | 4271 | h['connected'] = False |
| 4272 | h['serial'] = None |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4273 | |
Anas Nashif | 4fa82c4 | 2020-12-15 10:08:28 -0500 | [diff] [blame] | 4274 | for _detected in self.detected: |
| 4275 | for h in hwm: |
| 4276 | if _detected.id == h['id'] and _detected.product == h['product'] and not _detected.match: |
| 4277 | h['connected'] = True |
| 4278 | h['serial'] = _detected.serial |
| 4279 | _detected.match = True |
| 4280 | |
| 4281 | new_duts = list(filter(lambda d: not d.match, self.detected)) |
| 4282 | new = [] |
| 4283 | for d in new_duts: |
| 4284 | new.append(d.to_dict()) |
| 4285 | |
| 4286 | if hwm: |
| 4287 | hwm = hwm + new |
| 4288 | else: |
| 4289 | hwm = new |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4290 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4291 | with open(hwm_file, 'w') as yaml_file: |
Anas Nashif | ae61b7e | 2020-07-06 11:30:55 -0400 | [diff] [blame] | 4292 | yaml.dump(hwm, yaml_file, Dumper=Dumper, default_flow_style=False) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4293 | |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4294 | self.load(hwm_file) |
| 4295 | logger.info("Registered devices:") |
| 4296 | self.dump() |
| 4297 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4298 | else: |
| 4299 | # create new file |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4300 | dl = [] |
| 4301 | for _connected in self.detected: |
| 4302 | platform = _connected.platform |
| 4303 | id = _connected.id |
| 4304 | runner = _connected.runner |
| 4305 | serial = _connected.serial |
| 4306 | product = _connected.product |
| 4307 | d = { |
| 4308 | 'platform': platform, |
| 4309 | 'id': id, |
| 4310 | 'runner': runner, |
| 4311 | 'serial': serial, |
Anas Nashif | 0e64b87 | 2021-05-17 09:45:03 -0400 | [diff] [blame] | 4312 | 'product': product, |
| 4313 | 'connected': _connected.connected |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4314 | } |
| 4315 | dl.append(d) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4316 | with open(hwm_file, 'w') as yaml_file: |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4317 | yaml.dump(dl, yaml_file, Dumper=Dumper, default_flow_style=False) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4318 | logger.info("Detected devices:") |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4319 | self.dump(detected=True) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4320 | |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4321 | def dump(self, filtered=[], header=[], connected_only=False, detected=False): |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4322 | print("") |
| 4323 | table = [] |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4324 | if detected: |
| 4325 | to_show = self.detected |
| 4326 | else: |
Anas Nashif | 8305d1b | 2020-11-26 11:55:02 -0500 | [diff] [blame] | 4327 | to_show = self.duts |
Anas Nashif | ad70a69 | 2020-11-26 09:19:41 -0500 | [diff] [blame] | 4328 | |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4329 | if not header: |
| 4330 | header = ["Platform", "ID", "Serial device"] |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4331 | for p in to_show: |
| 4332 | platform = p.platform |
| 4333 | connected = p.connected |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4334 | if filtered and platform not in filtered: |
| 4335 | continue |
| 4336 | |
| 4337 | if not connected_only or connected: |
Anas Nashif | 531fe89 | 2020-09-11 13:56:33 -0400 | [diff] [blame] | 4338 | table.append([platform, p.id, p.serial]) |
Anas Nashif | ce2b418 | 2020-03-24 14:40:28 -0400 | [diff] [blame] | 4339 | |
| 4340 | print(tabulate(table, headers=header, tablefmt="github")) |