blob: c71831bdef8744f20369fefabff3cbc59bd789a8 [file] [log] [blame]
Anas Nashifce2b4182020-03-24 14:40:28 -04001#!/usr/bin/env python3
2# vim: set syntax=python ts=4 :
3#
4# Copyright (c) 2018 Intel Corporation
5# SPDX-License-Identifier: Apache-2.0
Anas Nashifce2b4182020-03-24 14:40:28 -04006import os
7import contextlib
8import string
9import mmap
10import sys
11import re
12import subprocess
13import select
14import shutil
15import shlex
16import signal
17import threading
18import concurrent.futures
19from collections import OrderedDict
Anas Nashifce2b4182020-03-24 14:40:28 -040020import queue
21import time
22import csv
23import glob
24import concurrent
25import xml.etree.ElementTree as ET
26import logging
Anas Nashifce2b4182020-03-24 14:40:28 -040027from pathlib import Path
28from distutils.spawn import find_executable
29from colorama import Fore
Martí Bolívar9c92baa2020-07-08 14:43:07 -070030import pickle
Martí Bolívar07dce822020-04-13 16:50:51 -070031import platform
Anas Nashifae61b7e2020-07-06 11:30:55 -040032import yaml
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -070033import json
Anas Nashif531fe892020-09-11 13:56:33 -040034from multiprocessing import Lock, Process, Value
Yuval Peressdee79d22021-07-23 11:47:52 -060035from typing import List
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -070036
Anas Nashifae61b7e2020-07-06 11:30:55 -040037try:
38 # Use the C LibYAML parser if available, rather than the Python parser.
39 # It's much faster.
Anas Nashifae61b7e2020-07-06 11:30:55 -040040 from yaml import CSafeLoader as SafeLoader
41 from yaml import CDumper as Dumper
42except ImportError:
Martí Bolívard8698cb2020-07-08 14:55:14 -070043 from yaml import SafeLoader, Dumper
Anas Nashifce2b4182020-03-24 14:40:28 -040044
45try:
46 import serial
47except ImportError:
48 print("Install pyserial python module with pip to use --device-testing option.")
49
50try:
51 from tabulate import tabulate
52except ImportError:
53 print("Install tabulate python module with pip to use --device-testing option.")
54
Wentong Wu0d619ae2020-05-05 19:46:49 -040055try:
56 import psutil
57except ImportError:
Anas Nashif77946fa2020-05-21 18:19:01 -040058 print("Install psutil python module with pip to run in Qemu.")
Wentong Wu0d619ae2020-05-05 19:46:49 -040059
Piotr Golyzniak8b773482021-11-02 16:13:26 +010060try:
61 import pty
62except ImportError as capture_error:
63 if os.name == "nt": # "nt" means that program is running on Windows OS
64 pass # "--device-serial-pty" option is not supported on Windows OS
65 else:
66 raise capture_error
67
Anas Nashifce2b4182020-03-24 14:40:28 -040068ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
69if not ZEPHYR_BASE:
70 sys.exit("$ZEPHYR_BASE environment variable undefined")
71
Martí Bolívar9c92baa2020-07-08 14:43:07 -070072# This is needed to load edt.pickle files.
Martí Bolívar53328472021-03-26 16:18:58 -070073sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts",
74 "python-devicetree", "src"))
75from devicetree import edtlib # pylint: disable=unused-import
Anas Nashifce2b4182020-03-24 14:40:28 -040076
77# Use this for internal comparisons; that's what canonicalization is
78# for. Don't use it when invoking other components of the build system
79# to avoid confusing and hard to trace inconsistencies in error messages
80# and logs, generated Makefiles, etc. compared to when users invoke these
81# components directly.
82# Note "normalization" is different from canonicalization, see os.path.
83canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE)
84
85sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/"))
86
Anas Nashife508bab2020-12-07 11:27:32 -050087import scl
88import expr_parser
Anas Nashifce2b4182020-03-24 14:40:28 -040089
Anas Nashifb18f3112020-12-07 11:40:19 -050090logger = logging.getLogger('twister')
Anas Nashifce2b4182020-03-24 14:40:28 -040091logger.setLevel(logging.DEBUG)
92
Anas Nashif531fe892020-09-11 13:56:33 -040093
94class ExecutionCounter(object):
95 def __init__(self, total=0):
96 self._done = Value('i', 0)
97 self._passed = Value('i', 0)
Anas Nashif3b939da2020-11-24 13:21:27 -050098 self._skipped_configs = Value('i', 0)
99 self._skipped_runtime = Value('i', 0)
100 self._skipped_cases = Value('i', 0)
Anas Nashif531fe892020-09-11 13:56:33 -0400101 self._error = Value('i', 0)
102 self._failed = Value('i', 0)
103 self._total = Value('i', total)
104 self._cases = Value('i', 0)
Anas Nashif3b939da2020-11-24 13:21:27 -0500105
Anas Nashif531fe892020-09-11 13:56:33 -0400106
107 self.lock = Lock()
108
109 @property
110 def cases(self):
111 with self._cases.get_lock():
112 return self._cases.value
113
114 @cases.setter
115 def cases(self, value):
116 with self._cases.get_lock():
117 self._cases.value = value
118
119 @property
120 def skipped_cases(self):
121 with self._skipped_cases.get_lock():
122 return self._skipped_cases.value
123
124 @skipped_cases.setter
125 def skipped_cases(self, value):
126 with self._skipped_cases.get_lock():
127 self._skipped_cases.value = value
128
129 @property
130 def error(self):
131 with self._error.get_lock():
132 return self._error.value
133
134 @error.setter
135 def error(self, value):
136 with self._error.get_lock():
137 self._error.value = value
138
139 @property
140 def done(self):
141 with self._done.get_lock():
142 return self._done.value
143
144 @done.setter
145 def done(self, value):
146 with self._done.get_lock():
147 self._done.value = value
148
149 @property
150 def passed(self):
151 with self._passed.get_lock():
152 return self._passed.value
153
154 @passed.setter
155 def passed(self, value):
156 with self._passed.get_lock():
157 self._passed.value = value
158
159 @property
Anas Nashif3b939da2020-11-24 13:21:27 -0500160 def skipped_configs(self):
161 with self._skipped_configs.get_lock():
162 return self._skipped_configs.value
Anas Nashif531fe892020-09-11 13:56:33 -0400163
Anas Nashif3b939da2020-11-24 13:21:27 -0500164 @skipped_configs.setter
165 def skipped_configs(self, value):
166 with self._skipped_configs.get_lock():
167 self._skipped_configs.value = value
Anas Nashif531fe892020-09-11 13:56:33 -0400168
169 @property
Anas Nashif3b939da2020-11-24 13:21:27 -0500170 def skipped_runtime(self):
171 with self._skipped_runtime.get_lock():
172 return self._skipped_runtime.value
Anas Nashif531fe892020-09-11 13:56:33 -0400173
Anas Nashif3b939da2020-11-24 13:21:27 -0500174 @skipped_runtime.setter
175 def skipped_runtime(self, value):
176 with self._skipped_runtime.get_lock():
177 self._skipped_runtime.value = value
Anas Nashif531fe892020-09-11 13:56:33 -0400178
179 @property
180 def failed(self):
181 with self._failed.get_lock():
182 return self._failed.value
183
184 @failed.setter
185 def failed(self, value):
186 with self._failed.get_lock():
187 self._failed.value = value
188
189 @property
190 def total(self):
191 with self._total.get_lock():
192 return self._total.value
Anas Nashifce2b4182020-03-24 14:40:28 -0400193
194class CMakeCacheEntry:
195 '''Represents a CMake cache entry.
196
197 This class understands the type system in a CMakeCache.txt, and
198 converts the following cache types to Python types:
199
200 Cache Type Python type
201 ---------- -------------------------------------------
202 FILEPATH str
203 PATH str
204 STRING str OR list of str (if ';' is in the value)
205 BOOL bool
206 INTERNAL str OR list of str (if ';' is in the value)
207 ---------- -------------------------------------------
208 '''
209
210 # Regular expression for a cache entry.
211 #
212 # CMake variable names can include escape characters, allowing a
213 # wider set of names than is easy to match with a regular
214 # expression. To be permissive here, use a non-greedy match up to
215 # the first colon (':'). This breaks if the variable name has a
216 # colon inside, but it's good enough.
217 CACHE_ENTRY = re.compile(
218 r'''(?P<name>.*?) # name
219 :(?P<type>FILEPATH|PATH|STRING|BOOL|INTERNAL) # type
220 =(?P<value>.*) # value
221 ''', re.X)
222
223 @classmethod
224 def _to_bool(cls, val):
225 # Convert a CMake BOOL string into a Python bool.
226 #
227 # "True if the constant is 1, ON, YES, TRUE, Y, or a
228 # non-zero number. False if the constant is 0, OFF, NO,
229 # FALSE, N, IGNORE, NOTFOUND, the empty string, or ends in
230 # the suffix -NOTFOUND. Named boolean constants are
231 # case-insensitive. If the argument is not one of these
232 # constants, it is treated as a variable."
233 #
234 # https://cmake.org/cmake/help/v3.0/command/if.html
235 val = val.upper()
236 if val in ('ON', 'YES', 'TRUE', 'Y'):
237 return 1
238 elif val in ('OFF', 'NO', 'FALSE', 'N', 'IGNORE', 'NOTFOUND', ''):
239 return 0
240 elif val.endswith('-NOTFOUND'):
241 return 0
242 else:
243 try:
244 v = int(val)
245 return v != 0
246 except ValueError as exc:
247 raise ValueError('invalid bool {}'.format(val)) from exc
248
249 @classmethod
250 def from_line(cls, line, line_no):
251 # Comments can only occur at the beginning of a line.
252 # (The value of an entry could contain a comment character).
253 if line.startswith('//') or line.startswith('#'):
254 return None
255
256 # Whitespace-only lines do not contain cache entries.
257 if not line.strip():
258 return None
259
260 m = cls.CACHE_ENTRY.match(line)
261 if not m:
262 return None
263
264 name, type_, value = (m.group(g) for g in ('name', 'type', 'value'))
265 if type_ == 'BOOL':
266 try:
267 value = cls._to_bool(value)
268 except ValueError as exc:
269 args = exc.args + ('on line {}: {}'.format(line_no, line),)
270 raise ValueError(args) from exc
271 elif type_ in ['STRING', 'INTERNAL']:
272 # If the value is a CMake list (i.e. is a string which
273 # contains a ';'), convert to a Python list.
274 if ';' in value:
275 value = value.split(';')
276
277 return CMakeCacheEntry(name, value)
278
279 def __init__(self, name, value):
280 self.name = name
281 self.value = value
282
283 def __str__(self):
284 fmt = 'CMakeCacheEntry(name={}, value={})'
285 return fmt.format(self.name, self.value)
286
287
288class CMakeCache:
289 '''Parses and represents a CMake cache file.'''
290
291 @staticmethod
292 def from_file(cache_file):
293 return CMakeCache(cache_file)
294
295 def __init__(self, cache_file):
296 self.cache_file = cache_file
297 self.load(cache_file)
298
299 def load(self, cache_file):
300 entries = []
301 with open(cache_file, 'r') as cache:
302 for line_no, line in enumerate(cache):
303 entry = CMakeCacheEntry.from_line(line, line_no)
304 if entry:
305 entries.append(entry)
306 self._entries = OrderedDict((e.name, e) for e in entries)
307
308 def get(self, name, default=None):
309 entry = self._entries.get(name)
310 if entry is not None:
311 return entry.value
312 else:
313 return default
314
315 def get_list(self, name, default=None):
316 if default is None:
317 default = []
318 entry = self._entries.get(name)
319 if entry is not None:
320 value = entry.value
321 if isinstance(value, list):
322 return value
323 elif isinstance(value, str):
324 return [value] if value else []
325 else:
326 msg = 'invalid value {} type {}'
327 raise RuntimeError(msg.format(value, type(value)))
328 else:
329 return default
330
331 def __contains__(self, name):
332 return name in self._entries
333
334 def __getitem__(self, name):
335 return self._entries[name].value
336
337 def __setitem__(self, name, entry):
338 if not isinstance(entry, CMakeCacheEntry):
339 msg = 'improper type {} for value {}, expecting CMakeCacheEntry'
340 raise TypeError(msg.format(type(entry), entry))
341 self._entries[name] = entry
342
343 def __delitem__(self, name):
344 del self._entries[name]
345
346 def __iter__(self):
347 return iter(self._entries.values())
348
349
Anas Nashif45943702020-12-11 17:55:15 -0500350class TwisterException(Exception):
Anas Nashifce2b4182020-03-24 14:40:28 -0400351 pass
352
353
Anas Nashif45943702020-12-11 17:55:15 -0500354class TwisterRuntimeError(TwisterException):
Anas Nashifce2b4182020-03-24 14:40:28 -0400355 pass
356
357
Anas Nashif45943702020-12-11 17:55:15 -0500358class ConfigurationError(TwisterException):
Anas Nashifce2b4182020-03-24 14:40:28 -0400359 def __init__(self, cfile, message):
Anas Nashif45943702020-12-11 17:55:15 -0500360 TwisterException.__init__(self, cfile + ": " + message)
Anas Nashifce2b4182020-03-24 14:40:28 -0400361
362
Anas Nashif45943702020-12-11 17:55:15 -0500363class BuildError(TwisterException):
Anas Nashifce2b4182020-03-24 14:40:28 -0400364 pass
365
366
Anas Nashif45943702020-12-11 17:55:15 -0500367class ExecutionError(TwisterException):
Anas Nashifce2b4182020-03-24 14:40:28 -0400368 pass
369
370
371class HarnessImporter:
372
373 def __init__(self, name):
Anas Nashife508bab2020-12-07 11:27:32 -0500374 sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
Anas Nashifce2b4182020-03-24 14:40:28 -0400375 module = __import__("harness")
376 if name:
377 my_class = getattr(module, name)
378 else:
379 my_class = getattr(module, "Test")
380
381 self.instance = my_class()
382
383
384class Handler:
385 def __init__(self, instance, type_str="build"):
386 """Constructor
387
388 """
Anas Nashifce2b4182020-03-24 14:40:28 -0400389 self.state = "waiting"
390 self.run = False
391 self.duration = 0
392 self.type_str = type_str
393
394 self.binary = None
395 self.pid_fn = None
396 self.call_make_run = False
397
398 self.name = instance.name
399 self.instance = instance
400 self.timeout = instance.testcase.timeout
401 self.sourcedir = instance.testcase.source_dir
402 self.build_dir = instance.build_dir
403 self.log = os.path.join(self.build_dir, "handler.log")
404 self.returncode = 0
405 self.set_state("running", self.duration)
406 self.generator = None
407 self.generator_cmd = None
408
409 self.args = []
Jingru Wangfed1c542021-06-11 11:54:55 +0800410 self.terminated = False
Anas Nashifce2b4182020-03-24 14:40:28 -0400411
412 def set_state(self, state, duration):
Anas Nashifce2b4182020-03-24 14:40:28 -0400413 self.state = state
414 self.duration = duration
Anas Nashifce2b4182020-03-24 14:40:28 -0400415
416 def get_state(self):
Anas Nashifce2b4182020-03-24 14:40:28 -0400417 ret = (self.state, self.duration)
Anas Nashifce2b4182020-03-24 14:40:28 -0400418 return ret
419
420 def record(self, harness):
421 if harness.recording:
422 filename = os.path.join(self.build_dir, "recording.csv")
423 with open(filename, "at") as csvfile:
424 cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep)
425 cw.writerow(harness.fieldnames)
426 for instance in harness.recording:
427 cw.writerow(instance)
428
Jingru Wangfed1c542021-06-11 11:54:55 +0800429 def terminate(self, proc):
430 # encapsulate terminate functionality so we do it consistently where ever
431 # we might want to terminate the proc. We need try_kill_process_by_pid
432 # because of both how newer ninja (1.6.0 or greater) and .NET / renode
433 # work. Newer ninja's don't seem to pass SIGTERM down to the children
434 # so we need to use try_kill_process_by_pid.
435 for child in psutil.Process(proc.pid).children(recursive=True):
436 try:
437 os.kill(child.pid, signal.SIGTERM)
438 except ProcessLookupError:
439 pass
440 proc.terminate()
441 # sleep for a while before attempting to kill
442 time.sleep(0.5)
443 proc.kill()
444 self.terminated = True
445
Piotr Golyzniak0e1ecb32021-09-23 00:14:04 +0200446 def add_missing_testscases(self, harness):
447 """
448 If testsuite was broken by some error (e.g. timeout) it is necessary to
449 add information about next testcases, which were not be
450 performed due to this error.
451 """
452 for c in self.instance.testcase.cases:
453 if c not in harness.tests:
454 harness.tests[c] = "BLOCK"
455
Anas Nashifce2b4182020-03-24 14:40:28 -0400456
457class BinaryHandler(Handler):
458 def __init__(self, instance, type_str):
459 """Constructor
460
461 @param instance Test Instance
462 """
463 super().__init__(instance, type_str)
464
Eugeniy Paltsev16032b672020-10-08 22:42:02 +0300465 self.call_west_flash = False
Anas Nashifce2b4182020-03-24 14:40:28 -0400466
467 # Tool options
468 self.valgrind = False
469 self.lsan = False
470 self.asan = False
Christian Taedcke3dbe9f22020-07-06 16:00:57 +0200471 self.ubsan = False
Anas Nashifce2b4182020-03-24 14:40:28 -0400472 self.coverage = False
473
474 def try_kill_process_by_pid(self):
475 if self.pid_fn:
476 pid = int(open(self.pid_fn).read())
477 os.unlink(self.pid_fn)
478 self.pid_fn = None # clear so we don't try to kill the binary twice
479 try:
480 os.kill(pid, signal.SIGTERM)
481 except ProcessLookupError:
482 pass
483
Anas Nashif531fe892020-09-11 13:56:33 -0400484 def _output_reader(self, proc):
485 self.line = proc.stdout.readline()
486
487 def _output_handler(self, proc, harness):
YouhuaX Zhu965c8b92020-10-22 09:37:27 +0800488 if harness.is_pytest:
489 harness.handle(None)
490 return
491
Anas Nashifce2b4182020-03-24 14:40:28 -0400492 log_out_fp = open(self.log, "wt")
Anas Nashif531fe892020-09-11 13:56:33 -0400493 timeout_extended = False
494 timeout_time = time.time() + self.timeout
495 while True:
496 this_timeout = timeout_time - time.time()
497 if this_timeout < 0:
Anas Nashifce2b4182020-03-24 14:40:28 -0400498 break
Anas Nashif531fe892020-09-11 13:56:33 -0400499 reader_t = threading.Thread(target=self._output_reader, args=(proc,), daemon=True)
500 reader_t.start()
501 reader_t.join(this_timeout)
502 if not reader_t.is_alive():
503 line = self.line
504 logger.debug("OUTPUT: {0}".format(line.decode('utf-8').rstrip()))
505 log_out_fp.write(line.decode('utf-8'))
506 log_out_fp.flush()
507 harness.handle(line.decode('utf-8').rstrip())
508 if harness.state:
509 if not timeout_extended or harness.capture_coverage:
510 timeout_extended = True
511 if harness.capture_coverage:
512 timeout_time = time.time() + 30
513 else:
514 timeout_time = time.time() + 2
515 else:
516 reader_t.join(0)
517 break
518 try:
519 # POSIX arch based ztests end on their own,
520 # so let's give it up to 100ms to do so
521 proc.wait(0.1)
522 except subprocess.TimeoutExpired:
523 self.terminate(proc)
Anas Nashifce2b4182020-03-24 14:40:28 -0400524
525 log_out_fp.close()
526
527 def handle(self):
528
529 harness_name = self.instance.testcase.harness.capitalize()
530 harness_import = HarnessImporter(harness_name)
531 harness = harness_import.instance
532 harness.configure(self.instance)
533
534 if self.call_make_run:
535 command = [self.generator_cmd, "run"]
Eugeniy Paltsev16032b672020-10-08 22:42:02 +0300536 elif self.call_west_flash:
537 command = ["west", "flash", "--skip-rebuild", "-d", self.build_dir]
Anas Nashifce2b4182020-03-24 14:40:28 -0400538 else:
539 command = [self.binary]
540
541 run_valgrind = False
542 if self.valgrind and shutil.which("valgrind"):
543 command = ["valgrind", "--error-exitcode=2",
544 "--leak-check=full",
545 "--suppressions=" + ZEPHYR_BASE + "/scripts/valgrind.supp",
546 "--log-file=" + self.build_dir + "/valgrind.log"
547 ] + command
548 run_valgrind = True
549
550 logger.debug("Spawning process: " +
551 " ".join(shlex.quote(word) for word in command) + os.linesep +
552 "in directory: " + self.build_dir)
553
554 start_time = time.time()
555
556 env = os.environ.copy()
557 if self.asan:
558 env["ASAN_OPTIONS"] = "log_path=stdout:" + \
559 env.get("ASAN_OPTIONS", "")
560 if not self.lsan:
561 env["ASAN_OPTIONS"] += "detect_leaks=0"
562
Christian Taedcke3dbe9f22020-07-06 16:00:57 +0200563 if self.ubsan:
564 env["UBSAN_OPTIONS"] = "log_path=stdout:halt_on_error=1:" + \
565 env.get("UBSAN_OPTIONS", "")
566
Anas Nashifce2b4182020-03-24 14:40:28 -0400567 with subprocess.Popen(command, stdout=subprocess.PIPE,
568 stderr=subprocess.PIPE, cwd=self.build_dir, env=env) as proc:
569 logger.debug("Spawning BinaryHandler Thread for %s" % self.name)
Anas Nashif531fe892020-09-11 13:56:33 -0400570 t = threading.Thread(target=self._output_handler, args=(proc, harness,), daemon=True)
Anas Nashifce2b4182020-03-24 14:40:28 -0400571 t.start()
Anas Nashif531fe892020-09-11 13:56:33 -0400572 t.join()
Anas Nashifce2b4182020-03-24 14:40:28 -0400573 if t.is_alive():
574 self.terminate(proc)
575 t.join()
576 proc.wait()
577 self.returncode = proc.returncode
Eugeniy Paltsev79b3a502020-12-22 20:00:01 +0300578 self.try_kill_process_by_pid()
Anas Nashifce2b4182020-03-24 14:40:28 -0400579
580 handler_time = time.time() - start_time
581
582 if self.coverage:
583 subprocess.call(["GCOV_PREFIX=" + self.build_dir,
584 "gcov", self.sourcedir, "-b", "-s", self.build_dir], shell=True)
585
Anas Nashifce2b4182020-03-24 14:40:28 -0400586 # FIXME: This is needed when killing the simulator, the console is
587 # garbled and needs to be reset. Did not find a better way to do that.
Eugeniy Paltsevafb34e32020-11-29 21:28:28 +0300588 if sys.stdout.isatty():
589 subprocess.call(["stty", "sane"])
Anas Nashifce2b4182020-03-24 14:40:28 -0400590
YouhuaX Zhu965c8b92020-10-22 09:37:27 +0800591 if harness.is_pytest:
592 harness.pytest_run(self.log)
Anas Nashifce2b4182020-03-24 14:40:28 -0400593 self.instance.results = harness.tests
594
595 if not self.terminated and self.returncode != 0:
596 # When a process is killed, the default handler returns 128 + SIGTERM
597 # so in that case the return code itself is not meaningful
598 self.set_state("failed", handler_time)
599 self.instance.reason = "Failed"
600 elif run_valgrind and self.returncode == 2:
601 self.set_state("failed", handler_time)
602 self.instance.reason = "Valgrind error"
603 elif harness.state:
604 self.set_state(harness.state, handler_time)
Anas Nashifb802af82020-04-26 21:57:38 -0400605 if harness.state == "failed":
606 self.instance.reason = "Failed"
Anas Nashifce2b4182020-03-24 14:40:28 -0400607 else:
608 self.set_state("timeout", handler_time)
609 self.instance.reason = "Timeout"
Piotr Golyzniak0e1ecb32021-09-23 00:14:04 +0200610 self.add_missing_testscases(harness)
Anas Nashifce2b4182020-03-24 14:40:28 -0400611
612 self.record(harness)
613
614
615class DeviceHandler(Handler):
616
617 def __init__(self, instance, type_str):
618 """Constructor
619
620 @param instance Test Instance
621 """
622 super().__init__(instance, type_str)
623
624 self.suite = None
Anas Nashifce2b4182020-03-24 14:40:28 -0400625
626 def monitor_serial(self, ser, halt_fileno, harness):
YouhuaX Zhu965c8b92020-10-22 09:37:27 +0800627 if harness.is_pytest:
628 harness.handle(None)
629 return
630
Anas Nashifce2b4182020-03-24 14:40:28 -0400631 log_out_fp = open(self.log, "wt")
632
633 ser_fileno = ser.fileno()
634 readlist = [halt_fileno, ser_fileno]
635
Andrei Emeltchenko10fa48d2021-01-12 09:56:32 +0200636 if self.coverage:
637 # Set capture_coverage to True to indicate that right after
638 # test results we should get coverage data, otherwise we exit
639 # from the test.
640 harness.capture_coverage = True
641
642 ser.flush()
643
Anas Nashifce2b4182020-03-24 14:40:28 -0400644 while ser.isOpen():
645 readable, _, _ = select.select(readlist, [], [], self.timeout)
646
647 if halt_fileno in readable:
648 logger.debug('halted')
649 ser.close()
650 break
651 if ser_fileno not in readable:
652 continue # Timeout.
653
654 serial_line = None
655 try:
656 serial_line = ser.readline()
657 except TypeError:
658 pass
659 except serial.SerialException:
660 ser.close()
661 break
662
663 # Just because ser_fileno has data doesn't mean an entire line
664 # is available yet.
665 if serial_line:
666 sl = serial_line.decode('utf-8', 'ignore').lstrip()
667 logger.debug("DEVICE: {0}".format(sl.rstrip()))
668
669 log_out_fp.write(sl)
670 log_out_fp.flush()
671 harness.handle(sl.rstrip())
672
673 if harness.state:
Andrei Emeltchenko10fa48d2021-01-12 09:56:32 +0200674 if not harness.capture_coverage:
675 ser.close()
676 break
Anas Nashifce2b4182020-03-24 14:40:28 -0400677
678 log_out_fp.close()
679
Anas Nashif3b86f132020-05-21 10:35:33 -0400680 def device_is_available(self, instance):
681 device = instance.platform.name
682 fixture = instance.testcase.harness_config.get("fixture")
Anas Nashif8305d1b2020-11-26 11:55:02 -0500683 for d in self.suite.duts:
Anas Nashif531fe892020-09-11 13:56:33 -0400684 if fixture and fixture not in d.fixtures:
Anas Nashif3b86f132020-05-21 10:35:33 -0400685 continue
Andy Ross098fce32021-03-02 05:13:07 -0800686 if d.platform != device or not (d.serial or d.serial_pty):
687 continue
688 d.lock.acquire()
689 avail = False
690 if d.available:
Anas Nashif531fe892020-09-11 13:56:33 -0400691 d.available = 0
692 d.counter += 1
Andy Ross098fce32021-03-02 05:13:07 -0800693 avail = True
694 d.lock.release()
695 if avail:
Anas Nashif2a740532021-02-04 08:08:20 -0500696 return d
Anas Nashifce2b4182020-03-24 14:40:28 -0400697
Anas Nashif2a740532021-02-04 08:08:20 -0500698 return None
Anas Nashifce2b4182020-03-24 14:40:28 -0400699
700 def make_device_available(self, serial):
Anas Nashif8305d1b2020-11-26 11:55:02 -0500701 for d in self.suite.duts:
Anas Nashif531fe892020-09-11 13:56:33 -0400702 if d.serial == serial or d.serial_pty:
703 d.available = 1
Anas Nashifce2b4182020-03-24 14:40:28 -0400704
705 @staticmethod
706 def run_custom_script(script, timeout):
707 with subprocess.Popen(script, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as proc:
708 try:
709 stdout, _ = proc.communicate(timeout=timeout)
710 logger.debug(stdout.decode())
711
712 except subprocess.TimeoutExpired:
713 proc.kill()
714 proc.communicate()
715 logger.error("{} timed out".format(script))
716
717 def handle(self):
718 out_state = "failed"
Anas Nashif531fe892020-09-11 13:56:33 -0400719 runner = None
Anas Nashifce2b4182020-03-24 14:40:28 -0400720
Anas Nashif2a740532021-02-04 08:08:20 -0500721 hardware = self.device_is_available(self.instance)
722 while not hardware:
Anas Nashifce2b4182020-03-24 14:40:28 -0400723 logger.debug("Waiting for device {} to become available".format(self.instance.platform.name))
724 time.sleep(1)
Anas Nashif2a740532021-02-04 08:08:20 -0500725 hardware = self.device_is_available(self.instance)
Anas Nashifce2b4182020-03-24 14:40:28 -0400726
Anas Nashif2a740532021-02-04 08:08:20 -0500727 runner = hardware.runner or self.suite.west_runner
Anas Nashif531fe892020-09-11 13:56:33 -0400728 serial_pty = hardware.serial_pty
Anas Nashif2a740532021-02-04 08:08:20 -0500729
Anas Nashif531fe892020-09-11 13:56:33 -0400730 ser_pty_process = None
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300731 if serial_pty:
732 master, slave = pty.openpty()
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300733 try:
Andrei Emeltchenko9f7a9032020-09-02 11:43:07 +0300734 ser_pty_process = subprocess.Popen(re.split(',| ', serial_pty), stdout=master, stdin=master, stderr=master)
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300735 except subprocess.CalledProcessError as error:
736 logger.error("Failed to run subprocess {}, error {}".format(serial_pty, error.output))
737 return
738
739 serial_device = os.ttyname(slave)
740 else:
Anas Nashif531fe892020-09-11 13:56:33 -0400741 serial_device = hardware.serial
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300742
Dennis Rufferc714c782021-10-05 15:34:54 -0700743 logger.debug("Using serial device {} @ {} baud".format(serial_device, hardware.serial_baud))
Anas Nashifce2b4182020-03-24 14:40:28 -0400744
Øyvind Rønningstadf72aef12020-07-01 16:58:54 +0200745 if (self.suite.west_flash is not None) or runner:
746 command = ["west", "flash", "--skip-rebuild", "-d", self.build_dir]
747 command_extra_args = []
748
749 # There are three ways this option is used.
750 # 1) bare: --west-flash
751 # This results in options.west_flash == []
752 # 2) with a value: --west-flash="--board-id=42"
753 # This results in options.west_flash == "--board-id=42"
754 # 3) Multiple values: --west-flash="--board-id=42,--erase"
755 # This results in options.west_flash == "--board-id=42 --erase"
756 if self.suite.west_flash and self.suite.west_flash != []:
757 command_extra_args.extend(self.suite.west_flash.split(','))
758
759 if runner:
760 command.append("--runner")
761 command.append(runner)
762
Anas Nashif531fe892020-09-11 13:56:33 -0400763 board_id = hardware.probe_id or hardware.id
764 product = hardware.product
Øyvind Rønningstadf72aef12020-07-01 16:58:54 +0200765 if board_id is not None:
766 if runner == "pyocd":
767 command_extra_args.append("--board-id")
768 command_extra_args.append(board_id)
769 elif runner == "nrfjprog":
770 command_extra_args.append("--snr")
771 command_extra_args.append(board_id)
772 elif runner == "openocd" and product == "STM32 STLink":
773 command_extra_args.append("--cmd-pre-init")
774 command_extra_args.append("hla_serial %s" % (board_id))
775 elif runner == "openocd" and product == "STLINK-V3":
776 command_extra_args.append("--cmd-pre-init")
777 command_extra_args.append("hla_serial %s" % (board_id))
778 elif runner == "openocd" and product == "EDBG CMSIS-DAP":
779 command_extra_args.append("--cmd-pre-init")
780 command_extra_args.append("cmsis_dap_serial %s" % (board_id))
781 elif runner == "jlink":
782 command.append("--tool-opt=-SelectEmuBySN %s" % (board_id))
783
784 if command_extra_args != []:
785 command.append('--')
786 command.extend(command_extra_args)
787 else:
788 command = [self.generator_cmd, "-C", self.build_dir, "flash"]
789
Anas Nashif531fe892020-09-11 13:56:33 -0400790 pre_script = hardware.pre_script
791 post_flash_script = hardware.post_flash_script
792 post_script = hardware.post_script
Watson Zeng3b43d942020-08-25 15:22:39 +0800793
794 if pre_script:
795 self.run_custom_script(pre_script, 30)
796
Anas Nashifce2b4182020-03-24 14:40:28 -0400797 try:
798 ser = serial.Serial(
799 serial_device,
Dennis Rufferc714c782021-10-05 15:34:54 -0700800 baudrate=hardware.serial_baud,
Anas Nashifce2b4182020-03-24 14:40:28 -0400801 parity=serial.PARITY_NONE,
802 stopbits=serial.STOPBITS_ONE,
803 bytesize=serial.EIGHTBITS,
804 timeout=self.timeout
805 )
806 except serial.SerialException as e:
807 self.set_state("failed", 0)
808 self.instance.reason = "Failed"
809 logger.error("Serial device error: %s" % (str(e)))
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300810
Anas Nashif531fe892020-09-11 13:56:33 -0400811 if serial_pty and ser_pty_process:
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300812 ser_pty_process.terminate()
813 outs, errs = ser_pty_process.communicate()
814 logger.debug("Process {} terminated outs: {} errs {}".format(serial_pty, outs, errs))
815
Anas Nashifce2b4182020-03-24 14:40:28 -0400816 self.make_device_available(serial_device)
817 return
818
819 ser.flush()
820
821 harness_name = self.instance.testcase.harness.capitalize()
822 harness_import = HarnessImporter(harness_name)
823 harness = harness_import.instance
824 harness.configure(self.instance)
825 read_pipe, write_pipe = os.pipe()
826 start_time = time.time()
827
Anas Nashifce2b4182020-03-24 14:40:28 -0400828 t = threading.Thread(target=self.monitor_serial, daemon=True,
829 args=(ser, read_pipe, harness))
830 t.start()
831
832 d_log = "{}/device.log".format(self.instance.build_dir)
833 logger.debug('Flash command: %s', command)
834 try:
835 stdout = stderr = None
836 with subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as proc:
837 try:
838 (stdout, stderr) = proc.communicate(timeout=30)
839 logger.debug(stdout.decode())
840
841 if proc.returncode != 0:
842 self.instance.reason = "Device issue (Flash?)"
843 with open(d_log, "w") as dlog_fp:
844 dlog_fp.write(stderr.decode())
Maciej Perkowskif050a992021-02-24 13:43:05 +0100845 os.write(write_pipe, b'x') # halt the thread
846 out_state = "flash_error"
Anas Nashifce2b4182020-03-24 14:40:28 -0400847 except subprocess.TimeoutExpired:
848 proc.kill()
849 (stdout, stderr) = proc.communicate()
850 self.instance.reason = "Device issue (Timeout)"
851
852 with open(d_log, "w") as dlog_fp:
853 dlog_fp.write(stderr.decode())
854
855 except subprocess.CalledProcessError:
856 os.write(write_pipe, b'x') # halt the thread
857
858 if post_flash_script:
859 self.run_custom_script(post_flash_script, 30)
860
Anas Nashifce2b4182020-03-24 14:40:28 -0400861 t.join(self.timeout)
862 if t.is_alive():
863 logger.debug("Timed out while monitoring serial output on {}".format(self.instance.platform.name))
864 out_state = "timeout"
865
866 if ser.isOpen():
867 ser.close()
868
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300869 if serial_pty:
870 ser_pty_process.terminate()
871 outs, errs = ser_pty_process.communicate()
872 logger.debug("Process {} terminated outs: {} errs {}".format(serial_pty, outs, errs))
873
Anas Nashifce2b4182020-03-24 14:40:28 -0400874 os.close(write_pipe)
875 os.close(read_pipe)
876
877 handler_time = time.time() - start_time
878
Maciej Perkowskif050a992021-02-24 13:43:05 +0100879 if out_state in ["timeout", "flash_error"]:
Piotr Golyzniak0e1ecb32021-09-23 00:14:04 +0200880 self.add_missing_testscases(harness)
Anas Nashifce2b4182020-03-24 14:40:28 -0400881
Maciej Perkowskif050a992021-02-24 13:43:05 +0100882 if out_state == "timeout":
883 self.instance.reason = "Timeout"
884 elif out_state == "flash_error":
885 self.instance.reason = "Flash error"
Anas Nashifce2b4182020-03-24 14:40:28 -0400886
YouhuaX Zhu965c8b92020-10-22 09:37:27 +0800887 if harness.is_pytest:
888 harness.pytest_run(self.log)
Anas Nashifce2b4182020-03-24 14:40:28 -0400889 self.instance.results = harness.tests
890
peng1 chen64787352021-01-27 11:00:05 +0800891 # sometimes a test instance hasn't been executed successfully with an
892 # empty dictionary results, in order to include it into final report,
893 # so fill the results as BLOCK
894 if self.instance.results == {}:
895 for k in self.instance.testcase.cases:
896 self.instance.results[k] = 'BLOCK'
897
Anas Nashifce2b4182020-03-24 14:40:28 -0400898 if harness.state:
899 self.set_state(harness.state, handler_time)
Maciej Perkowskif050a992021-02-24 13:43:05 +0100900 if harness.state == "failed":
Anas Nashifce2b4182020-03-24 14:40:28 -0400901 self.instance.reason = "Failed"
902 else:
903 self.set_state(out_state, handler_time)
904
905 if post_script:
906 self.run_custom_script(post_script, 30)
907
908 self.make_device_available(serial_device)
Anas Nashifce2b4182020-03-24 14:40:28 -0400909 self.record(harness)
910
911
912class QEMUHandler(Handler):
913 """Spawns a thread to monitor QEMU output from pipes
914
915 We pass QEMU_PIPE to 'make run' and monitor the pipes for output.
916 We need to do this as once qemu starts, it runs forever until killed.
917 Test cases emit special messages to the console as they run, we check
918 for these to collect whether the test passed or failed.
919 """
920
921 def __init__(self, instance, type_str):
922 """Constructor
923
924 @param instance Test instance
925 """
926
927 super().__init__(instance, type_str)
928 self.fifo_fn = os.path.join(instance.build_dir, "qemu-fifo")
929
930 self.pid_fn = os.path.join(instance.build_dir, "qemu.pid")
931
Daniel Leungfaae15d2020-08-18 10:13:35 -0700932 if "ignore_qemu_crash" in instance.testcase.tags:
933 self.ignore_qemu_crash = True
934 self.ignore_unexpected_eof = True
935 else:
936 self.ignore_qemu_crash = False
937 self.ignore_unexpected_eof = False
938
Anas Nashifce2b4182020-03-24 14:40:28 -0400939 @staticmethod
Wentong Wu0d619ae2020-05-05 19:46:49 -0400940 def _get_cpu_time(pid):
941 """get process CPU time.
942
943 The guest virtual time in QEMU icount mode isn't host time and
944 it's maintained by counting guest instructions, so we use QEMU
945 process exection time to mostly simulate the time of guest OS.
946 """
947 proc = psutil.Process(pid)
948 cpu_time = proc.cpu_times()
949 return cpu_time.user + cpu_time.system
950
951 @staticmethod
Daniel Leungfaae15d2020-08-18 10:13:35 -0700952 def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results, harness,
953 ignore_unexpected_eof=False):
Anas Nashifce2b4182020-03-24 14:40:28 -0400954 fifo_in = fifo_fn + ".in"
955 fifo_out = fifo_fn + ".out"
956
957 # These in/out nodes are named from QEMU's perspective, not ours
958 if os.path.exists(fifo_in):
959 os.unlink(fifo_in)
960 os.mkfifo(fifo_in)
961 if os.path.exists(fifo_out):
962 os.unlink(fifo_out)
963 os.mkfifo(fifo_out)
964
965 # We don't do anything with out_fp but we need to open it for
966 # writing so that QEMU doesn't block, due to the way pipes work
967 out_fp = open(fifo_in, "wb")
968 # Disable internal buffering, we don't
969 # want read() or poll() to ever block if there is data in there
970 in_fp = open(fifo_out, "rb", buffering=0)
971 log_out_fp = open(logfile, "wt")
972
973 start_time = time.time()
974 timeout_time = start_time + timeout
975 p = select.poll()
976 p.register(in_fp, select.POLLIN)
977 out_state = None
978
979 line = ""
980 timeout_extended = False
Wentong Wu0d619ae2020-05-05 19:46:49 -0400981
982 pid = 0
983 if os.path.exists(pid_fn):
984 pid = int(open(pid_fn).read())
985
Anas Nashifce2b4182020-03-24 14:40:28 -0400986 while True:
987 this_timeout = int((timeout_time - time.time()) * 1000)
988 if this_timeout < 0 or not p.poll(this_timeout):
Wentong Wu517633c2020-07-24 21:13:01 +0800989 try:
990 if pid and this_timeout > 0:
991 #there's possibility we polled nothing because
992 #of not enough CPU time scheduled by host for
993 #QEMU process during p.poll(this_timeout)
994 cpu_time = QEMUHandler._get_cpu_time(pid)
995 if cpu_time < timeout and not out_state:
996 timeout_time = time.time() + (timeout - cpu_time)
997 continue
998 except ProcessLookupError:
999 out_state = "failed"
1000 break
Wentong Wu0d619ae2020-05-05 19:46:49 -04001001
Anas Nashifce2b4182020-03-24 14:40:28 -04001002 if not out_state:
1003 out_state = "timeout"
1004 break
1005
Wentong Wu0d619ae2020-05-05 19:46:49 -04001006 if pid == 0 and os.path.exists(pid_fn):
1007 pid = int(open(pid_fn).read())
1008
YouhuaX Zhu965c8b92020-10-22 09:37:27 +08001009 if harness.is_pytest:
1010 harness.handle(None)
1011 out_state = harness.state
1012 break
1013
Anas Nashifce2b4182020-03-24 14:40:28 -04001014 try:
1015 c = in_fp.read(1).decode("utf-8")
1016 except UnicodeDecodeError:
1017 # Test is writing something weird, fail
1018 out_state = "unexpected byte"
1019 break
1020
1021 if c == "":
1022 # EOF, this shouldn't happen unless QEMU crashes
Daniel Leungfaae15d2020-08-18 10:13:35 -07001023 if not ignore_unexpected_eof:
1024 out_state = "unexpected eof"
Anas Nashifce2b4182020-03-24 14:40:28 -04001025 break
1026 line = line + c
1027 if c != "\n":
1028 continue
1029
1030 # line contains a full line of data output from QEMU
1031 log_out_fp.write(line)
1032 log_out_fp.flush()
1033 line = line.strip()
Anas Nashif743594f2020-09-10 08:19:47 -04001034 logger.debug(f"QEMU ({pid}): {line}")
Anas Nashifce2b4182020-03-24 14:40:28 -04001035
1036 harness.handle(line)
1037 if harness.state:
1038 # if we have registered a fail make sure the state is not
1039 # overridden by a false success message coming from the
1040 # testsuite
Anas Nashif869ca052020-07-07 14:29:07 -04001041 if out_state not in ['failed', 'unexpected eof', 'unexpected byte']:
Anas Nashifce2b4182020-03-24 14:40:28 -04001042 out_state = harness.state
1043
1044 # if we get some state, that means test is doing well, we reset
1045 # the timeout and wait for 2 more seconds to catch anything
1046 # printed late. We wait much longer if code
1047 # coverage is enabled since dumping this information can
1048 # take some time.
1049 if not timeout_extended or harness.capture_coverage:
1050 timeout_extended = True
1051 if harness.capture_coverage:
1052 timeout_time = time.time() + 30
1053 else:
1054 timeout_time = time.time() + 2
1055 line = ""
1056
YouhuaX Zhu965c8b92020-10-22 09:37:27 +08001057 if harness.is_pytest:
1058 harness.pytest_run(logfile)
1059 out_state = harness.state
1060
Anas Nashifce2b4182020-03-24 14:40:28 -04001061 handler.record(harness)
1062
1063 handler_time = time.time() - start_time
Anas Nashif743594f2020-09-10 08:19:47 -04001064 logger.debug(f"QEMU ({pid}) complete ({out_state}) after {handler_time} seconds")
Anas Nashif869ca052020-07-07 14:29:07 -04001065
Anas Nashifce2b4182020-03-24 14:40:28 -04001066 if out_state == "timeout":
1067 handler.instance.reason = "Timeout"
Anas Nashif06052922020-07-15 22:44:24 -04001068 handler.set_state("failed", handler_time)
Anas Nashifce2b4182020-03-24 14:40:28 -04001069 elif out_state == "failed":
1070 handler.instance.reason = "Failed"
Anas Nashif869ca052020-07-07 14:29:07 -04001071 handler.set_state("failed", handler_time)
Anas Nashif06052922020-07-15 22:44:24 -04001072 elif out_state in ['unexpected eof', 'unexpected byte']:
Anas Nashif869ca052020-07-07 14:29:07 -04001073 handler.instance.reason = out_state
Anas Nashif06052922020-07-15 22:44:24 -04001074 handler.set_state("failed", handler_time)
1075 else:
1076 handler.set_state(out_state, handler_time)
Anas Nashifce2b4182020-03-24 14:40:28 -04001077
1078 log_out_fp.close()
1079 out_fp.close()
1080 in_fp.close()
Wentong Wu0d619ae2020-05-05 19:46:49 -04001081 if pid:
Anas Nashifce2b4182020-03-24 14:40:28 -04001082 try:
1083 if pid:
1084 os.kill(pid, signal.SIGTERM)
1085 except ProcessLookupError:
1086 # Oh well, as long as it's dead! User probably sent Ctrl-C
1087 pass
1088
1089 os.unlink(fifo_in)
1090 os.unlink(fifo_out)
1091
1092 def handle(self):
1093 self.results = {}
1094 self.run = True
1095
1096 # We pass this to QEMU which looks for fifos with .in and .out
1097 # suffixes.
Anas Nashifce2b4182020-03-24 14:40:28 -04001098
Anas Nashifc1c10992020-09-10 07:36:00 -04001099 self.fifo_fn = os.path.join(self.instance.build_dir, "qemu-fifo")
Anas Nashifce2b4182020-03-24 14:40:28 -04001100 self.pid_fn = os.path.join(self.instance.build_dir, "qemu.pid")
Anas Nashifc1c10992020-09-10 07:36:00 -04001101
Anas Nashifce2b4182020-03-24 14:40:28 -04001102 if os.path.exists(self.pid_fn):
1103 os.unlink(self.pid_fn)
1104
1105 self.log_fn = self.log
1106
1107 harness_import = HarnessImporter(self.instance.testcase.harness.capitalize())
1108 harness = harness_import.instance
1109 harness.configure(self.instance)
YouhuaX Zhu965c8b92020-10-22 09:37:27 +08001110
Anas Nashifce2b4182020-03-24 14:40:28 -04001111 self.thread = threading.Thread(name=self.name, target=QEMUHandler._thread,
1112 args=(self, self.timeout, self.build_dir,
1113 self.log_fn, self.fifo_fn,
Daniel Leungfaae15d2020-08-18 10:13:35 -07001114 self.pid_fn, self.results, harness,
1115 self.ignore_unexpected_eof))
Anas Nashifce2b4182020-03-24 14:40:28 -04001116
1117 self.instance.results = harness.tests
1118 self.thread.daemon = True
1119 logger.debug("Spawning QEMUHandler Thread for %s" % self.name)
1120 self.thread.start()
Eugeniy Paltsevafb34e32020-11-29 21:28:28 +03001121 if sys.stdout.isatty():
1122 subprocess.call(["stty", "sane"])
Anas Nashifce2b4182020-03-24 14:40:28 -04001123
1124 logger.debug("Running %s (%s)" % (self.name, self.type_str))
1125 command = [self.generator_cmd]
1126 command += ["-C", self.build_dir, "run"]
1127
Anas Nashiffdc02b62020-09-09 19:11:19 -04001128 is_timeout = False
Anas Nashif743594f2020-09-10 08:19:47 -04001129 qemu_pid = None
Anas Nashiffdc02b62020-09-09 19:11:19 -04001130
Anas Nashifce2b4182020-03-24 14:40:28 -04001131 with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.build_dir) as proc:
1132 logger.debug("Spawning QEMUHandler Thread for %s" % self.name)
Anas Nashif743594f2020-09-10 08:19:47 -04001133
Wentong Wu7ec57b42020-05-05 19:19:18 -04001134 try:
1135 proc.wait(self.timeout)
1136 except subprocess.TimeoutExpired:
Anas Nashiffdc02b62020-09-09 19:11:19 -04001137 # sometimes QEMU can't handle SIGTERM signal correctly
1138 # in that case kill -9 QEMU process directly and leave
Anas Nashifb18f3112020-12-07 11:40:19 -05001139 # twister to judge testing result by console output
Anas Nashiffdc02b62020-09-09 19:11:19 -04001140
1141 is_timeout = True
Jingru Wangfed1c542021-06-11 11:54:55 +08001142 self.terminate(proc)
1143 if harness.state == "passed":
1144 self.returncode = 0
Wentong Wu7ec57b42020-05-05 19:19:18 -04001145 else:
Wentong Wu7ec57b42020-05-05 19:19:18 -04001146 self.returncode = proc.returncode
1147 else:
Anas Nashif743594f2020-09-10 08:19:47 -04001148 if os.path.exists(self.pid_fn):
1149 qemu_pid = int(open(self.pid_fn).read())
1150 logger.debug(f"No timeout, return code from QEMU ({qemu_pid}): {proc.returncode}")
Wentong Wu7ec57b42020-05-05 19:19:18 -04001151 self.returncode = proc.returncode
Daniel Leung5b1b4a32020-08-18 10:10:36 -07001152 # Need to wait for harness to finish processing
1153 # output from QEMU. Otherwise it might miss some
1154 # error messages.
Jingru Wangfed1c542021-06-11 11:54:55 +08001155 self.thread.join(0)
1156 if self.thread.is_alive():
1157 logger.debug("Timed out while monitoring QEMU output")
Daniel Leung5b1b4a32020-08-18 10:10:36 -07001158
Wentong Wu7ec57b42020-05-05 19:19:18 -04001159 if os.path.exists(self.pid_fn):
Anas Nashif743594f2020-09-10 08:19:47 -04001160 qemu_pid = int(open(self.pid_fn).read())
Wentong Wu7ec57b42020-05-05 19:19:18 -04001161 os.unlink(self.pid_fn)
Anas Nashifce2b4182020-03-24 14:40:28 -04001162
Anas Nashif743594f2020-09-10 08:19:47 -04001163 logger.debug(f"return code from QEMU ({qemu_pid}): {self.returncode}")
Anas Nashif869ca052020-07-07 14:29:07 -04001164
Daniel Leungfaae15d2020-08-18 10:13:35 -07001165 if (self.returncode != 0 and not self.ignore_qemu_crash) or not harness.state:
Anas Nashifce2b4182020-03-24 14:40:28 -04001166 self.set_state("failed", 0)
Anas Nashiffdc02b62020-09-09 19:11:19 -04001167 if is_timeout:
1168 self.instance.reason = "Timeout"
1169 else:
1170 self.instance.reason = "Exited with {}".format(self.returncode)
Piotr Golyzniak0e1ecb32021-09-23 00:14:04 +02001171 self.add_missing_testscases(harness)
Anas Nashifce2b4182020-03-24 14:40:28 -04001172
1173 def get_fifo(self):
1174 return self.fifo_fn
1175
1176
1177class SizeCalculator:
1178 alloc_sections = [
1179 "bss",
1180 "noinit",
1181 "app_bss",
1182 "app_noinit",
1183 "ccm_bss",
1184 "ccm_noinit"
1185 ]
1186
1187 rw_sections = [
1188 "datas",
1189 "initlevel",
1190 "exceptions",
1191 "initshell",
Andrew Boie45979da2020-05-23 14:38:39 -07001192 "_static_thread_data_area",
1193 "k_timer_area",
1194 "k_mem_slab_area",
1195 "k_mem_pool_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001196 "sw_isr_table",
Andrew Boie45979da2020-05-23 14:38:39 -07001197 "k_sem_area",
1198 "k_mutex_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001199 "app_shmem_regions",
1200 "_k_fifo_area",
1201 "_k_lifo_area",
Andrew Boie45979da2020-05-23 14:38:39 -07001202 "k_stack_area",
1203 "k_msgq_area",
1204 "k_mbox_area",
1205 "k_pipe_area",
Daniel Leung203556c2020-08-24 12:40:26 -07001206 "net_if_area",
1207 "net_if_dev_area",
1208 "net_l2_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001209 "net_l2_data",
Andrew Boie45979da2020-05-23 14:38:39 -07001210 "k_queue_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001211 "_net_buf_pool_area",
1212 "app_datas",
1213 "kobject_data",
1214 "mmu_tables",
1215 "app_pad",
1216 "priv_stacks",
1217 "ccm_data",
1218 "usb_descriptor",
1219 "usb_data", "usb_bos_desc",
Jukka Rissanen420b1952020-04-01 12:47:53 +03001220 "uart_mux",
Anas Nashifce2b4182020-03-24 14:40:28 -04001221 'log_backends_sections',
1222 'log_dynamic_sections',
1223 'log_const_sections',
1224 "app_smem",
1225 'shell_root_cmds_sections',
1226 'log_const_sections',
1227 "font_entry_sections",
1228 "priv_stacks_noinit",
1229 "_GCOV_BSS_SECTION_NAME",
1230 "gcov",
Daniel Leung203556c2020-08-24 12:40:26 -07001231 "nocache",
1232 "devices",
1233 "k_heap_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001234 ]
1235
1236 # These get copied into RAM only on non-XIP
1237 ro_sections = [
1238 "rom_start",
1239 "text",
1240 "ctors",
1241 "init_array",
1242 "reset",
Andrew Boie45979da2020-05-23 14:38:39 -07001243 "z_object_assignment_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001244 "rodata",
Anas Nashifce2b4182020-03-24 14:40:28 -04001245 "net_l2",
1246 "vector",
1247 "sw_isr_table",
Andrew Boie45979da2020-05-23 14:38:39 -07001248 "settings_handler_static_area",
Daniel Leung203556c2020-08-24 12:40:26 -07001249 "bt_l2cap_fixed_chan_area",
1250 "bt_l2cap_br_fixed_chan_area",
1251 "bt_gatt_service_static_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001252 "vectors",
Andrew Boie45979da2020-05-23 14:38:39 -07001253 "net_socket_register_area",
1254 "net_ppp_proto",
1255 "shell_area",
1256 "tracing_backend_area",
Daniel Leung203556c2020-08-24 12:40:26 -07001257 "ppp_protocol_handler_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001258 ]
1259
1260 def __init__(self, filename, extra_sections):
1261 """Constructor
1262
1263 @param filename Path to the output binary
1264 The <filename> is parsed by objdump to determine section sizes
1265 """
1266 # Make sure this is an ELF binary
1267 with open(filename, "rb") as f:
1268 magic = f.read(4)
1269
1270 try:
1271 if magic != b'\x7fELF':
Anas Nashif45943702020-12-11 17:55:15 -05001272 raise TwisterRuntimeError("%s is not an ELF binary" % filename)
Anas Nashifce2b4182020-03-24 14:40:28 -04001273 except Exception as e:
1274 print(str(e))
1275 sys.exit(2)
1276
1277 # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
1278 # GREP can not be used as it returns an error if the symbol is not
1279 # found.
1280 is_xip_command = "nm " + filename + \
1281 " | awk '/CONFIG_XIP/ { print $3 }'"
1282 is_xip_output = subprocess.check_output(
1283 is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
1284 "utf-8").strip()
1285 try:
1286 if is_xip_output.endswith("no symbols"):
Anas Nashif45943702020-12-11 17:55:15 -05001287 raise TwisterRuntimeError("%s has no symbol information" % filename)
Anas Nashifce2b4182020-03-24 14:40:28 -04001288 except Exception as e:
1289 print(str(e))
1290 sys.exit(2)
1291
1292 self.is_xip = (len(is_xip_output) != 0)
1293
1294 self.filename = filename
1295 self.sections = []
1296 self.rom_size = 0
1297 self.ram_size = 0
1298 self.extra_sections = extra_sections
1299
1300 self._calculate_sizes()
1301
1302 def get_ram_size(self):
1303 """Get the amount of RAM the application will use up on the device
1304
1305 @return amount of RAM, in bytes
1306 """
1307 return self.ram_size
1308
1309 def get_rom_size(self):
1310 """Get the size of the data that this application uses on device's flash
1311
1312 @return amount of ROM, in bytes
1313 """
1314 return self.rom_size
1315
1316 def unrecognized_sections(self):
1317 """Get a list of sections inside the binary that weren't recognized
1318
1319 @return list of unrecognized section names
1320 """
1321 slist = []
1322 for v in self.sections:
1323 if not v["recognized"]:
1324 slist.append(v["name"])
1325 return slist
1326
1327 def _calculate_sizes(self):
1328 """ Calculate RAM and ROM usage by section """
1329 objdump_command = "objdump -h " + self.filename
1330 objdump_output = subprocess.check_output(
1331 objdump_command, shell=True).decode("utf-8").splitlines()
1332
1333 for line in objdump_output:
1334 words = line.split()
1335
1336 if not words: # Skip lines that are too short
1337 continue
1338
1339 index = words[0]
1340 if not index[0].isdigit(): # Skip lines that do not start
1341 continue # with a digit
1342
1343 name = words[1] # Skip lines with section names
1344 if name[0] == '.': # starting with '.'
1345 continue
1346
1347 # TODO this doesn't actually reflect the size in flash or RAM as
1348 # it doesn't include linker-imposed padding between sections.
1349 # It is close though.
1350 size = int(words[2], 16)
1351 if size == 0:
1352 continue
1353
1354 load_addr = int(words[4], 16)
1355 virt_addr = int(words[3], 16)
1356
1357 # Add section to memory use totals (for both non-XIP and XIP scenarios)
1358 # Unrecognized section names are not included in the calculations.
1359 recognized = True
1360 if name in SizeCalculator.alloc_sections:
1361 self.ram_size += size
1362 stype = "alloc"
1363 elif name in SizeCalculator.rw_sections:
1364 self.ram_size += size
1365 self.rom_size += size
1366 stype = "rw"
1367 elif name in SizeCalculator.ro_sections:
1368 self.rom_size += size
1369 if not self.is_xip:
1370 self.ram_size += size
1371 stype = "ro"
1372 else:
1373 stype = "unknown"
1374 if name not in self.extra_sections:
1375 recognized = False
1376
1377 self.sections.append({"name": name, "load_addr": load_addr,
1378 "size": size, "virt_addr": virt_addr,
1379 "type": stype, "recognized": recognized})
1380
1381
1382
Anas Nashif45943702020-12-11 17:55:15 -05001383class TwisterConfigParser:
Anas Nashifce2b4182020-03-24 14:40:28 -04001384 """Class to read test case files with semantic checking
1385 """
1386
1387 def __init__(self, filename, schema):
Anas Nashif45943702020-12-11 17:55:15 -05001388 """Instantiate a new TwisterConfigParser object
Anas Nashifce2b4182020-03-24 14:40:28 -04001389
1390 @param filename Source .yaml file to read
1391 """
1392 self.data = {}
1393 self.schema = schema
1394 self.filename = filename
1395 self.tests = {}
1396 self.common = {}
1397
1398 def load(self):
1399 self.data = scl.yaml_load_verify(self.filename, self.schema)
1400
1401 if 'tests' in self.data:
1402 self.tests = self.data['tests']
1403 if 'common' in self.data:
1404 self.common = self.data['common']
1405
1406 def _cast_value(self, value, typestr):
1407 if isinstance(value, str):
1408 v = value.strip()
1409 if typestr == "str":
1410 return v
1411
1412 elif typestr == "float":
1413 return float(value)
1414
1415 elif typestr == "int":
1416 return int(value)
1417
1418 elif typestr == "bool":
1419 return value
1420
1421 elif typestr.startswith("list") and isinstance(value, list):
1422 return value
1423 elif typestr.startswith("list") and isinstance(value, str):
1424 vs = v.split()
1425 if len(typestr) > 4 and typestr[4] == ":":
1426 return [self._cast_value(vsi, typestr[5:]) for vsi in vs]
1427 else:
1428 return vs
1429
1430 elif typestr.startswith("set"):
1431 vs = v.split()
1432 if len(typestr) > 3 and typestr[3] == ":":
1433 return {self._cast_value(vsi, typestr[4:]) for vsi in vs}
1434 else:
1435 return set(vs)
1436
1437 elif typestr.startswith("map"):
1438 return value
1439 else:
1440 raise ConfigurationError(
1441 self.filename, "unknown type '%s'" % value)
1442
1443 def get_test(self, name, valid_keys):
1444 """Get a dictionary representing the keys/values within a test
1445
1446 @param name The test in the .yaml file to retrieve data from
1447 @param valid_keys A dictionary representing the intended semantics
1448 for this test. Each key in this dictionary is a key that could
1449 be specified, if a key is given in the .yaml file which isn't in
1450 here, it will generate an error. Each value in this dictionary
1451 is another dictionary containing metadata:
1452
1453 "default" - Default value if not given
1454 "type" - Data type to convert the text value to. Simple types
1455 supported are "str", "float", "int", "bool" which will get
1456 converted to respective Python data types. "set" and "list"
1457 may also be specified which will split the value by
1458 whitespace (but keep the elements as strings). finally,
1459 "list:<type>" and "set:<type>" may be given which will
1460 perform a type conversion after splitting the value up.
1461 "required" - If true, raise an error if not defined. If false
1462 and "default" isn't specified, a type conversion will be
1463 done on an empty string
1464 @return A dictionary containing the test key-value pairs with
1465 type conversion and default values filled in per valid_keys
1466 """
1467
1468 d = {}
1469 for k, v in self.common.items():
1470 d[k] = v
1471
1472 for k, v in self.tests[name].items():
Anas Nashifce2b4182020-03-24 14:40:28 -04001473 if k in d:
1474 if isinstance(d[k], str):
1475 # By default, we just concatenate string values of keys
1476 # which appear both in "common" and per-test sections,
1477 # but some keys are handled in adhoc way based on their
1478 # semantics.
1479 if k == "filter":
1480 d[k] = "(%s) and (%s)" % (d[k], v)
1481 else:
1482 d[k] += " " + v
1483 else:
1484 d[k] = v
1485
1486 for k, kinfo in valid_keys.items():
1487 if k not in d:
1488 if "required" in kinfo:
1489 required = kinfo["required"]
1490 else:
1491 required = False
1492
1493 if required:
1494 raise ConfigurationError(
1495 self.filename,
1496 "missing required value for '%s' in test '%s'" %
1497 (k, name))
1498 else:
1499 if "default" in kinfo:
1500 default = kinfo["default"]
1501 else:
1502 default = self._cast_value("", kinfo["type"])
1503 d[k] = default
1504 else:
1505 try:
1506 d[k] = self._cast_value(d[k], kinfo["type"])
1507 except ValueError:
1508 raise ConfigurationError(
1509 self.filename, "bad %s value '%s' for key '%s' in name '%s'" %
1510 (kinfo["type"], d[k], k, name))
1511
1512 return d
1513
1514
1515class Platform:
1516 """Class representing metadata for a particular platform
1517
1518 Maps directly to BOARD when building"""
1519
1520 platform_schema = scl.yaml_load(os.path.join(ZEPHYR_BASE,
Anas Nashifc24bf6f2020-12-07 11:18:07 -05001521 "scripts", "schemas", "twister", "platform-schema.yaml"))
Anas Nashifce2b4182020-03-24 14:40:28 -04001522
1523 def __init__(self):
1524 """Constructor.
1525
1526 """
1527
1528 self.name = ""
Anas Nashifb18f3112020-12-07 11:40:19 -05001529 self.twister = True
Anas Nashifce2b4182020-03-24 14:40:28 -04001530 # if no RAM size is specified by the board, take a default of 128K
1531 self.ram = 128
1532
1533 self.ignore_tags = []
Anas Nashife8e367a2020-07-16 16:27:04 -04001534 self.only_tags = []
Anas Nashifce2b4182020-03-24 14:40:28 -04001535 self.default = False
1536 # if no flash size is specified by the board, take a default of 512K
1537 self.flash = 512
1538 self.supported = set()
1539
1540 self.arch = ""
1541 self.type = "na"
1542 self.simulation = "na"
1543 self.supported_toolchains = []
1544 self.env = []
1545 self.env_satisfied = True
1546 self.filter_data = dict()
1547
1548 def load(self, platform_file):
Anas Nashif45943702020-12-11 17:55:15 -05001549 scp = TwisterConfigParser(platform_file, self.platform_schema)
Anas Nashifce2b4182020-03-24 14:40:28 -04001550 scp.load()
1551 data = scp.data
1552
1553 self.name = data['identifier']
Anas Nashifb18f3112020-12-07 11:40:19 -05001554 self.twister = data.get("twister", True)
Anas Nashifce2b4182020-03-24 14:40:28 -04001555 # if no RAM size is specified by the board, take a default of 128K
1556 self.ram = data.get("ram", 128)
1557 testing = data.get("testing", {})
1558 self.ignore_tags = testing.get("ignore_tags", [])
Anas Nashife8e367a2020-07-16 16:27:04 -04001559 self.only_tags = testing.get("only_tags", [])
Anas Nashifce2b4182020-03-24 14:40:28 -04001560 self.default = testing.get("default", False)
1561 # if no flash size is specified by the board, take a default of 512K
1562 self.flash = data.get("flash", 512)
1563 self.supported = set()
1564 for supp_feature in data.get("supported", []):
1565 for item in supp_feature.split(":"):
1566 self.supported.add(item)
1567
1568 self.arch = data['arch']
1569 self.type = data.get('type', "na")
1570 self.simulation = data.get('simulation', "na")
1571 self.supported_toolchains = data.get("toolchain", [])
1572 self.env = data.get("env", [])
1573 self.env_satisfied = True
1574 for env in self.env:
1575 if not os.environ.get(env, None):
1576 self.env_satisfied = False
1577
1578 def __repr__(self):
1579 return "<%s on %s>" % (self.name, self.arch)
1580
1581
Anas Nashifaff616d2020-04-17 21:24:57 -04001582class DisablePyTestCollectionMixin(object):
1583 __test__ = False
1584
1585
Yuval Peressdee79d22021-07-23 11:47:52 -06001586class ScanPathResult:
1587 """Result of the TestCase.scan_path function call.
1588
1589 Attributes:
1590 matches A list of test cases
1591 warnings A string containing one or more
1592 warnings to display
1593 has_registered_test_suites Whether or not the path contained any
1594 calls to the ztest_register_test_suite
1595 macro.
1596 has_run_registered_test_suites Whether or not the path contained at
1597 least one call to
1598 ztest_run_registered_test_suites.
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001599 has_test_main Whether or not the path contains a
1600 definition of test_main(void)
Yuval Peressdee79d22021-07-23 11:47:52 -06001601 """
1602 def __init__(self,
1603 matches: List[str] = None,
1604 warnings: str = None,
1605 has_registered_test_suites: bool = False,
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001606 has_run_registered_test_suites: bool = False,
1607 has_test_main: bool = False):
Yuval Peressdee79d22021-07-23 11:47:52 -06001608 self.matches = matches
1609 self.warnings = warnings
1610 self.has_registered_test_suites = has_registered_test_suites
1611 self.has_run_registered_test_suites = has_run_registered_test_suites
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001612 self.has_test_main = has_test_main
Yuval Peressdee79d22021-07-23 11:47:52 -06001613
1614 def __eq__(self, other):
1615 if not isinstance(other, ScanPathResult):
1616 return False
1617 return (sorted(self.matches) == sorted(other.matches) and
1618 self.warnings == other.warnings and
1619 (self.has_registered_test_suites ==
1620 other.has_registered_test_suites) and
1621 (self.has_run_registered_test_suites ==
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001622 other.has_run_registered_test_suites) and
1623 self.has_test_main == other.has_test_main)
Yuval Peressdee79d22021-07-23 11:47:52 -06001624
1625
Anas Nashifaff616d2020-04-17 21:24:57 -04001626class TestCase(DisablePyTestCollectionMixin):
Anas Nashifce2b4182020-03-24 14:40:28 -04001627 """Class representing a test application
1628 """
1629
Anas Nashifaff616d2020-04-17 21:24:57 -04001630 def __init__(self, testcase_root, workdir, name):
Anas Nashifce2b4182020-03-24 14:40:28 -04001631 """TestCase constructor.
1632
1633 This gets called by TestSuite as it finds and reads test yaml files.
1634 Multiple TestCase instances may be generated from a single testcase.yaml,
1635 each one corresponds to an entry within that file.
1636
1637 We need to have a unique name for every single test case. Since
1638 a testcase.yaml can define multiple tests, the canonical name for
1639 the test case is <workdir>/<name>.
1640
1641 @param testcase_root os.path.abspath() of one of the --testcase-root
1642 @param workdir Sub-directory of testcase_root where the
1643 .yaml test configuration file was found
1644 @param name Name of this test case, corresponding to the entry name
1645 in the test case configuration file. For many test cases that just
1646 define one test, can be anything and is usually "test". This is
1647 really only used to distinguish between different cases when
1648 the testcase.yaml defines multiple tests
Anas Nashifce2b4182020-03-24 14:40:28 -04001649 """
1650
Anas Nashifaff616d2020-04-17 21:24:57 -04001651
Anas Nashifce2b4182020-03-24 14:40:28 -04001652 self.source_dir = ""
1653 self.yamlfile = ""
1654 self.cases = []
Anas Nashifaff616d2020-04-17 21:24:57 -04001655 self.name = self.get_unique(testcase_root, workdir, name)
1656 self.id = name
Anas Nashifce2b4182020-03-24 14:40:28 -04001657
1658 self.type = None
Anas Nashifaff616d2020-04-17 21:24:57 -04001659 self.tags = set()
Anas Nashifce2b4182020-03-24 14:40:28 -04001660 self.extra_args = None
1661 self.extra_configs = None
Anas Nashifdca317c2020-08-26 11:28:25 -04001662 self.arch_allow = None
Anas Nashifce2b4182020-03-24 14:40:28 -04001663 self.arch_exclude = None
Anas Nashifaff616d2020-04-17 21:24:57 -04001664 self.skip = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001665 self.platform_exclude = None
Anas Nashifdca317c2020-08-26 11:28:25 -04001666 self.platform_allow = None
Anas Nashifce2b4182020-03-24 14:40:28 -04001667 self.toolchain_exclude = None
Anas Nashifdca317c2020-08-26 11:28:25 -04001668 self.toolchain_allow = None
Anas Nashifce2b4182020-03-24 14:40:28 -04001669 self.tc_filter = None
1670 self.timeout = 60
1671 self.harness = ""
1672 self.harness_config = {}
1673 self.build_only = True
1674 self.build_on_all = False
1675 self.slow = False
Anas Nashifaff616d2020-04-17 21:24:57 -04001676 self.min_ram = -1
Anas Nashifce2b4182020-03-24 14:40:28 -04001677 self.depends_on = None
Anas Nashifaff616d2020-04-17 21:24:57 -04001678 self.min_flash = -1
Anas Nashifce2b4182020-03-24 14:40:28 -04001679 self.extra_sections = None
Anas Nashif1636c312020-05-28 08:02:54 -04001680 self.integration_platforms = []
Anas Nashifce2b4182020-03-24 14:40:28 -04001681
1682 @staticmethod
1683 def get_unique(testcase_root, workdir, name):
1684
1685 canonical_testcase_root = os.path.realpath(testcase_root)
1686 if Path(canonical_zephyr_base) in Path(canonical_testcase_root).parents:
1687 # This is in ZEPHYR_BASE, so include path in name for uniqueness
1688 # FIXME: We should not depend on path of test for unique names.
1689 relative_tc_root = os.path.relpath(canonical_testcase_root,
1690 start=canonical_zephyr_base)
1691 else:
1692 relative_tc_root = ""
1693
1694 # workdir can be "."
1695 unique = os.path.normpath(os.path.join(relative_tc_root, workdir, name))
Anas Nashif7a691252020-05-07 07:47:51 -04001696 check = name.split(".")
1697 if len(check) < 2:
Anas Nashif45943702020-12-11 17:55:15 -05001698 raise TwisterException(f"""bad test name '{name}' in {testcase_root}/{workdir}. \
Anas Nashif7a691252020-05-07 07:47:51 -04001699Tests should reference the category and subsystem with a dot as a separator.
1700 """
1701 )
Anas Nashifce2b4182020-03-24 14:40:28 -04001702 return unique
1703
1704 @staticmethod
1705 def scan_file(inf_name):
1706 suite_regex = re.compile(
1707 # do not match until end-of-line, otherwise we won't allow
1708 # stc_regex below to catch the ones that are declared in the same
1709 # line--as we only search starting the end of this match
1710 br"^\s*ztest_test_suite\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
1711 re.MULTILINE)
Yuval Peressdee79d22021-07-23 11:47:52 -06001712 registered_suite_regex = re.compile(
1713 br"^\s*ztest_register_test_suite"
1714 br"\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
1715 re.MULTILINE)
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001716 # Checks if the file contains a definition of "void test_main(void)"
1717 # Since ztest provides a plain test_main implementation it is OK to:
1718 # 1. register test suites and not call the run function iff the test
1719 # doesn't have a custom test_main.
1720 # 2. register test suites and a custom test_main definition iff the test
1721 # also calls ztest_run_registered_test_suites.
1722 test_main_regex = re.compile(
1723 br"^\s*void\s+test_main\(void\)",
1724 re.MULTILINE)
Anas Nashifce2b4182020-03-24 14:40:28 -04001725 stc_regex = re.compile(
Yuval Peressdee79d22021-07-23 11:47:52 -06001726 br"""^\s* # empy space at the beginning is ok
Anas Nashifce2b4182020-03-24 14:40:28 -04001727 # catch the case where it is declared in the same sentence, e.g:
1728 #
1729 # ztest_test_suite(mutex_complex, ztest_user_unit_test(TESTNAME));
Yuval Peressdee79d22021-07-23 11:47:52 -06001730 # ztest_register_test_suite(n, p, ztest_user_unit_test(TESTNAME),
1731 (?:ztest_
1732 (?:test_suite\(|register_test_suite\([a-zA-Z0-9_]+\s*,\s*)
1733 [a-zA-Z0-9_]+\s*,\s*
1734 )?
Anas Nashifce2b4182020-03-24 14:40:28 -04001735 # Catch ztest[_user]_unit_test-[_setup_teardown](TESTNAME)
Yuval Peressdee79d22021-07-23 11:47:52 -06001736 ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?
Anas Nashifce2b4182020-03-24 14:40:28 -04001737 # Consume the argument that becomes the extra testcse
Yuval Peressdee79d22021-07-23 11:47:52 -06001738 \(\s*(?P<stc_name>[a-zA-Z0-9_]+)
Anas Nashifce2b4182020-03-24 14:40:28 -04001739 # _setup_teardown() variant has two extra arguments that we ignore
Yuval Peressdee79d22021-07-23 11:47:52 -06001740 (?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?
1741 \s*\)""",
Anas Nashifce2b4182020-03-24 14:40:28 -04001742 # We don't check how it finishes; we don't care
Yuval Peressdee79d22021-07-23 11:47:52 -06001743 re.MULTILINE | re.VERBOSE)
Anas Nashifce2b4182020-03-24 14:40:28 -04001744 suite_run_regex = re.compile(
1745 br"^\s*ztest_run_test_suite\((?P<suite_name>[a-zA-Z0-9_]+)\)",
1746 re.MULTILINE)
Yuval Peressdee79d22021-07-23 11:47:52 -06001747 registered_suite_run_regex = re.compile(
1748 br"^\s*ztest_run_registered_test_suites\("
1749 br"(\*+|&)?(?P<state_identifier>[a-zA-Z0-9_]+)\)",
1750 re.MULTILINE)
Anas Nashifce2b4182020-03-24 14:40:28 -04001751 achtung_regex = re.compile(
1752 br"(#ifdef|#endif)",
1753 re.MULTILINE)
1754 warnings = None
Yuval Peressdee79d22021-07-23 11:47:52 -06001755 has_registered_test_suites = False
1756 has_run_registered_test_suites = False
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001757 has_test_main = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001758
1759 with open(inf_name) as inf:
1760 if os.name == 'nt':
1761 mmap_args = {'fileno': inf.fileno(), 'length': 0, 'access': mmap.ACCESS_READ}
1762 else:
1763 mmap_args = {'fileno': inf.fileno(), 'length': 0, 'flags': mmap.MAP_PRIVATE, 'prot': mmap.PROT_READ,
1764 'offset': 0}
1765
1766 with contextlib.closing(mmap.mmap(**mmap_args)) as main_c:
Anas Nashifce2b4182020-03-24 14:40:28 -04001767 suite_regex_match = suite_regex.search(main_c)
Yuval Peressdee79d22021-07-23 11:47:52 -06001768 registered_suite_regex_match = registered_suite_regex.search(
1769 main_c)
1770
1771 if registered_suite_regex_match:
1772 has_registered_test_suites = True
1773 if registered_suite_run_regex.search(main_c):
1774 has_run_registered_test_suites = True
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001775 if test_main_regex.search(main_c):
1776 has_test_main = True
Yuval Peressdee79d22021-07-23 11:47:52 -06001777
1778 if not suite_regex_match and not has_registered_test_suites:
Anas Nashifce2b4182020-03-24 14:40:28 -04001779 # can't find ztest_test_suite, maybe a client, because
1780 # it includes ztest.h
Yuval Peressdee79d22021-07-23 11:47:52 -06001781 return ScanPathResult(
1782 matches=None,
1783 warnings=None,
1784 has_registered_test_suites=has_registered_test_suites,
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001785 has_run_registered_test_suites=has_run_registered_test_suites,
1786 has_test_main=has_test_main)
Anas Nashifce2b4182020-03-24 14:40:28 -04001787
1788 suite_run_match = suite_run_regex.search(main_c)
Yuval Peressdee79d22021-07-23 11:47:52 -06001789 if suite_regex_match and not suite_run_match:
Anas Nashifce2b4182020-03-24 14:40:28 -04001790 raise ValueError("can't find ztest_run_test_suite")
1791
Yuval Peressdee79d22021-07-23 11:47:52 -06001792 if suite_regex_match:
1793 search_start = suite_regex_match.end()
1794 else:
1795 search_start = registered_suite_regex_match.end()
1796
1797 if suite_run_match:
1798 search_end = suite_run_match.start()
1799 else:
1800 search_end = re.compile(br"\);", re.MULTILINE) \
1801 .search(main_c, search_start) \
1802 .end()
Anas Nashifce2b4182020-03-24 14:40:28 -04001803 achtung_matches = re.findall(
1804 achtung_regex,
Yuval Peressdee79d22021-07-23 11:47:52 -06001805 main_c[search_start:search_end])
Anas Nashifce2b4182020-03-24 14:40:28 -04001806 if achtung_matches:
1807 warnings = "found invalid %s in ztest_test_suite()" \
Spoorthy Priya Yeraboluad4d4fc2020-06-25 02:57:05 -07001808 % ", ".join(sorted({match.decode() for match in achtung_matches},reverse = True))
Anas Nashifce2b4182020-03-24 14:40:28 -04001809 _matches = re.findall(
1810 stc_regex,
Yuval Peressdee79d22021-07-23 11:47:52 -06001811 main_c[search_start:search_end])
Anas Nashif44f7ba02020-05-12 12:26:41 -04001812 for match in _matches:
1813 if not match.decode().startswith("test_"):
1814 warnings = "Found a test that does not start with test_"
Maciej Perkowski034d4f22020-08-03 14:11:11 +02001815 matches = [match.decode().replace("test_", "", 1) for match in _matches]
Yuval Peressdee79d22021-07-23 11:47:52 -06001816 return ScanPathResult(
1817 matches=matches,
1818 warnings=warnings,
1819 has_registered_test_suites=has_registered_test_suites,
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001820 has_run_registered_test_suites=has_run_registered_test_suites,
1821 has_test_main=has_test_main)
Anas Nashifce2b4182020-03-24 14:40:28 -04001822
1823 def scan_path(self, path):
1824 subcases = []
Yuval Peressdee79d22021-07-23 11:47:52 -06001825 has_registered_test_suites = False
1826 has_run_registered_test_suites = False
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001827 has_test_main = False
Anas Nashif91fd68d2020-05-08 07:22:58 -04001828 for filename in glob.glob(os.path.join(path, "src", "*.c*")):
Anas Nashifce2b4182020-03-24 14:40:28 -04001829 try:
Yuval Peressdee79d22021-07-23 11:47:52 -06001830 result: ScanPathResult = self.scan_file(filename)
1831 if result.warnings:
1832 logger.error("%s: %s" % (filename, result.warnings))
1833 raise TwisterRuntimeError(
1834 "%s: %s" % (filename, result.warnings))
1835 if result.matches:
1836 subcases += result.matches
1837 if result.has_registered_test_suites:
1838 has_registered_test_suites = True
1839 if result.has_run_registered_test_suites:
1840 has_run_registered_test_suites = True
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001841 if result.has_test_main:
1842 has_test_main = True
Anas Nashifce2b4182020-03-24 14:40:28 -04001843 except ValueError as e:
1844 logger.error("%s: can't find: %s" % (filename, e))
Anas Nashif61c6e2b2020-05-07 07:03:30 -04001845
Anas Nashifce2b4182020-03-24 14:40:28 -04001846 for filename in glob.glob(os.path.join(path, "*.c")):
1847 try:
Yuval Peressdee79d22021-07-23 11:47:52 -06001848 result: ScanPathResult = self.scan_file(filename)
1849 if result.warnings:
1850 logger.error("%s: %s" % (filename, result.warnings))
1851 if result.matches:
1852 subcases += result.matches
Anas Nashifce2b4182020-03-24 14:40:28 -04001853 except ValueError as e:
1854 logger.error("%s: can't find: %s" % (filename, e))
Yuval Peressdee79d22021-07-23 11:47:52 -06001855
Yuval Peress27f6a5e2021-08-01 22:12:44 -06001856 if (has_registered_test_suites and has_test_main and
1857 not has_run_registered_test_suites):
Yuval Peressdee79d22021-07-23 11:47:52 -06001858 warning = \
1859 "Found call to 'ztest_register_test_suite()' but no "\
1860 "call to 'ztest_run_registered_test_suites()'"
1861 logger.error(warning)
1862 raise TwisterRuntimeError(warning)
1863
Anas Nashifce2b4182020-03-24 14:40:28 -04001864 return subcases
1865
1866 def parse_subcases(self, test_path):
1867 results = self.scan_path(test_path)
1868 for sub in results:
1869 name = "{}.{}".format(self.id, sub)
1870 self.cases.append(name)
1871
1872 if not results:
1873 self.cases.append(self.id)
1874
1875 def __str__(self):
1876 return self.name
1877
1878
Anas Nashifaff616d2020-04-17 21:24:57 -04001879class TestInstance(DisablePyTestCollectionMixin):
Anas Nashifce2b4182020-03-24 14:40:28 -04001880 """Class representing the execution of a particular TestCase on a platform
1881
1882 @param test The TestCase object we want to build/execute
1883 @param platform Platform object that we want to build and run against
1884 @param base_outdir Base directory for all test results. The actual
1885 out directory used is <outdir>/<platform>/<test case name>
1886 """
1887
1888 def __init__(self, testcase, platform, outdir):
1889
1890 self.testcase = testcase
1891 self.platform = platform
1892
1893 self.status = None
1894 self.reason = "Unknown"
1895 self.metrics = dict()
1896 self.handler = None
1897 self.outdir = outdir
1898
1899 self.name = os.path.join(platform.name, testcase.name)
1900 self.build_dir = os.path.join(outdir, platform.name, testcase.name)
1901
Anas Nashifce2b4182020-03-24 14:40:28 -04001902 self.run = False
1903
1904 self.results = {}
1905
Anas Nashif531fe892020-09-11 13:56:33 -04001906 def __getstate__(self):
1907 d = self.__dict__.copy()
1908 return d
1909
1910 def __setstate__(self, d):
1911 self.__dict__.update(d)
1912
Anas Nashifce2b4182020-03-24 14:40:28 -04001913 def __lt__(self, other):
1914 return self.name < other.name
1915
Anas Nashif4ca0b952020-07-24 09:22:25 -04001916
Anas Nashif405f1b62020-07-27 12:27:13 -04001917 @staticmethod
1918 def testcase_runnable(testcase, fixtures):
Anas Nashif4ca0b952020-07-24 09:22:25 -04001919 can_run = False
1920 # console harness allows us to run the test and capture data.
YouhuaX Zhu965c8b92020-10-22 09:37:27 +08001921 if testcase.harness in [ 'console', 'ztest', 'pytest']:
Anas Nashif405f1b62020-07-27 12:27:13 -04001922 can_run = True
Anas Nashif4ca0b952020-07-24 09:22:25 -04001923 # if we have a fixture that is also being supplied on the
1924 # command-line, then we need to run the test, not just build it.
1925 fixture = testcase.harness_config.get('fixture')
1926 if fixture:
Anas Nashif405f1b62020-07-27 12:27:13 -04001927 can_run = (fixture in fixtures)
Anas Nashif4ca0b952020-07-24 09:22:25 -04001928
1929 elif testcase.harness:
1930 can_run = False
1931 else:
1932 can_run = True
1933
1934 return can_run
1935
1936
Anas Nashifaff616d2020-04-17 21:24:57 -04001937 # Global testsuite parameters
Anas Nashif405f1b62020-07-27 12:27:13 -04001938 def check_runnable(self, enable_slow=False, filter='buildable', fixtures=[]):
Anas Nashifce2b4182020-03-24 14:40:28 -04001939
1940 # right now we only support building on windows. running is still work
1941 # in progress.
1942 if os.name == 'nt':
Anas Nashif4ca0b952020-07-24 09:22:25 -04001943 return False
Anas Nashifce2b4182020-03-24 14:40:28 -04001944
1945 # we asked for build-only on the command line
Anas Nashif405f1b62020-07-27 12:27:13 -04001946 if self.testcase.build_only:
Anas Nashif4ca0b952020-07-24 09:22:25 -04001947 return False
Anas Nashifce2b4182020-03-24 14:40:28 -04001948
1949 # Do not run slow tests:
1950 skip_slow = self.testcase.slow and not enable_slow
1951 if skip_slow:
Anas Nashif4ca0b952020-07-24 09:22:25 -04001952 return False
Anas Nashifce2b4182020-03-24 14:40:28 -04001953
Anas Nashif4ca0b952020-07-24 09:22:25 -04001954 target_ready = bool(self.testcase.type == "unit" or \
Anas Nashifce2b4182020-03-24 14:40:28 -04001955 self.platform.type == "native" or \
Jaxson Han8af11d42021-02-24 10:23:46 +08001956 self.platform.simulation in ["mdb-nsim", "nsim", "renode", "qemu", "tsim", "armfvp"] or \
Anas Nashif405f1b62020-07-27 12:27:13 -04001957 filter == 'runnable')
Anas Nashifce2b4182020-03-24 14:40:28 -04001958
1959 if self.platform.simulation == "nsim":
1960 if not find_executable("nsimdrv"):
Anas Nashif4ca0b952020-07-24 09:22:25 -04001961 target_ready = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001962
Eugeniy Paltsev91d7ec52020-10-07 22:12:42 +03001963 if self.platform.simulation == "mdb-nsim":
Anas Nashif0f831b12020-09-03 12:33:16 -04001964 if not find_executable("mdb"):
Anas Nashif405f1b62020-07-27 12:27:13 -04001965 target_ready = False
Anas Nashif0f831b12020-09-03 12:33:16 -04001966
Anas Nashifce2b4182020-03-24 14:40:28 -04001967 if self.platform.simulation == "renode":
1968 if not find_executable("renode"):
Anas Nashif4ca0b952020-07-24 09:22:25 -04001969 target_ready = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001970
Martin Åbergc1077142020-10-19 18:21:52 +02001971 if self.platform.simulation == "tsim":
1972 if not find_executable("tsim-leon3"):
1973 target_ready = False
1974
Anas Nashif4ca0b952020-07-24 09:22:25 -04001975 testcase_runnable = self.testcase_runnable(self.testcase, fixtures)
Anas Nashifce2b4182020-03-24 14:40:28 -04001976
Anas Nashif4ca0b952020-07-24 09:22:25 -04001977 return testcase_runnable and target_ready
Anas Nashifce2b4182020-03-24 14:40:28 -04001978
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02001979 def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable_coverage=False, coverage_platform=[]):
Anas Nashifb18f3112020-12-07 11:40:19 -05001980 # Create this in a "twister/" subdirectory otherwise this
Anas Nashifce2b4182020-03-24 14:40:28 -04001981 # will pass this overlay to kconfig.py *twice* and kconfig.cmake
1982 # will silently give that second time precedence over any
1983 # --extra-args=CONFIG_*
Anas Nashifb18f3112020-12-07 11:40:19 -05001984 subdir = os.path.join(self.build_dir, "twister")
Anas Nashifce2b4182020-03-24 14:40:28 -04001985
Kumar Gala51d69312020-09-24 13:28:50 -05001986 content = ""
Anas Nashifce2b4182020-03-24 14:40:28 -04001987
Kumar Gala51d69312020-09-24 13:28:50 -05001988 if self.testcase.extra_configs:
1989 content = "\n".join(self.testcase.extra_configs)
Anas Nashifce2b4182020-03-24 14:40:28 -04001990
Kumar Gala51d69312020-09-24 13:28:50 -05001991 if enable_coverage:
1992 if platform.name in coverage_platform:
1993 content = content + "\nCONFIG_COVERAGE=y"
1994 content = content + "\nCONFIG_COVERAGE_DUMP=y"
Anas Nashifce2b4182020-03-24 14:40:28 -04001995
Kumar Gala51d69312020-09-24 13:28:50 -05001996 if enable_asan:
1997 if platform.type == "native":
1998 content = content + "\nCONFIG_ASAN=y"
Anas Nashifce2b4182020-03-24 14:40:28 -04001999
Kumar Gala51d69312020-09-24 13:28:50 -05002000 if enable_ubsan:
2001 if platform.type == "native":
2002 content = content + "\nCONFIG_UBSAN=y"
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002003
Kumar Gala51d69312020-09-24 13:28:50 -05002004 if content:
2005 os.makedirs(subdir, exist_ok=True)
2006 file = os.path.join(subdir, "testcase_extra.conf")
2007 with open(file, "w") as f:
2008 f.write(content)
2009
2010 return content
Anas Nashifce2b4182020-03-24 14:40:28 -04002011
2012 def calculate_sizes(self):
2013 """Get the RAM/ROM sizes of a test case.
2014
2015 This can only be run after the instance has been executed by
2016 MakeGenerator, otherwise there won't be any binaries to measure.
2017
2018 @return A SizeCalculator object
2019 """
2020 fns = glob.glob(os.path.join(self.build_dir, "zephyr", "*.elf"))
2021 fns.extend(glob.glob(os.path.join(self.build_dir, "zephyr", "*.exe")))
2022 fns = [x for x in fns if not x.endswith('_prebuilt.elf')]
2023 if len(fns) != 1:
2024 raise BuildError("Missing/multiple output ELF binary")
2025
2026 return SizeCalculator(fns[0], self.testcase.extra_sections)
2027
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002028 def fill_results_by_status(self):
2029 """Fills results according to self.status
2030
2031 The method is used to propagate the instance level status
2032 to the test cases inside. Useful when the whole instance is skipped
2033 and the info is required also at the test cases level for reporting.
2034 Should be used with caution, e.g. should not be used
2035 to fill all results with passes
2036 """
2037 status_to_verdict = {
2038 'skipped': 'SKIP',
2039 'error': 'BLOCK',
2040 'failure': 'FAILED'
2041 }
2042
2043 for k in self.results:
2044 self.results[k] = status_to_verdict[self.status]
2045
Anas Nashifce2b4182020-03-24 14:40:28 -04002046 def __repr__(self):
2047 return "<TestCase %s on %s>" % (self.testcase.name, self.platform.name)
2048
2049
2050class CMake():
2051 config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
2052 dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
2053
2054 def __init__(self, testcase, platform, source_dir, build_dir):
2055
2056 self.cwd = None
2057 self.capture_output = True
2058
2059 self.defconfig = {}
2060 self.cmake_cache = {}
2061
2062 self.instance = None
2063 self.testcase = testcase
2064 self.platform = platform
2065 self.source_dir = source_dir
2066 self.build_dir = build_dir
2067 self.log = "build.log"
2068 self.generator = None
2069 self.generator_cmd = None
2070
2071 def parse_generated(self):
2072 self.defconfig = {}
2073 return {}
2074
2075 def run_build(self, args=[]):
2076
2077 logger.debug("Building %s for %s" % (self.source_dir, self.platform.name))
2078
2079 cmake_args = []
2080 cmake_args.extend(args)
2081 cmake = shutil.which('cmake')
2082 cmd = [cmake] + cmake_args
2083 kwargs = dict()
2084
2085 if self.capture_output:
2086 kwargs['stdout'] = subprocess.PIPE
2087 # CMake sends the output of message() to stderr unless it's STATUS
2088 kwargs['stderr'] = subprocess.STDOUT
2089
2090 if self.cwd:
2091 kwargs['cwd'] = self.cwd
2092
2093 p = subprocess.Popen(cmd, **kwargs)
2094 out, _ = p.communicate()
2095
2096 results = {}
2097 if p.returncode == 0:
2098 msg = "Finished building %s for %s" % (self.source_dir, self.platform.name)
2099
2100 self.instance.status = "passed"
2101 results = {'msg': msg, "returncode": p.returncode, "instance": self.instance}
2102
2103 if out:
2104 log_msg = out.decode(sys.getdefaultencoding())
2105 with open(os.path.join(self.build_dir, self.log), "a") as log:
2106 log.write(log_msg)
2107
2108 else:
2109 return None
2110 else:
2111 # A real error occurred, raise an exception
Anas Nashif5a6e64f2020-10-30 13:01:50 -04002112 log_msg = ""
Anas Nashifce2b4182020-03-24 14:40:28 -04002113 if out:
2114 log_msg = out.decode(sys.getdefaultencoding())
2115 with open(os.path.join(self.build_dir, self.log), "a") as log:
2116 log.write(log_msg)
2117
2118 if log_msg:
Henrik Brix Andersen9aa4a702021-09-27 21:21:03 +02002119 res = re.findall("region `(FLASH|ROM|RAM|ICCM|DCCM|SRAM)' overflowed by", log_msg)
Anas Nashiff68146f2020-11-28 11:07:06 -05002120 if res and not self.overflow_as_errors:
Anas Nashifce2b4182020-03-24 14:40:28 -04002121 logger.debug("Test skipped due to {} Overflow".format(res[0]))
2122 self.instance.status = "skipped"
2123 self.instance.reason = "{} overflow".format(res[0])
2124 else:
Anas Nashiff04461e2020-06-29 10:07:02 -04002125 self.instance.status = "error"
Anas Nashifce2b4182020-03-24 14:40:28 -04002126 self.instance.reason = "Build failure"
2127
2128 results = {
2129 "returncode": p.returncode,
2130 "instance": self.instance,
2131 }
2132
2133 return results
2134
2135 def run_cmake(self, args=[]):
2136
Anas Nashif50925412020-07-16 17:25:19 -04002137 if self.warnings_as_errors:
2138 ldflags = "-Wl,--fatal-warnings"
2139 cflags = "-Werror"
2140 aflags = "-Wa,--fatal-warnings"
Martí Bolívarc4079e42021-07-30 15:43:27 -07002141 gen_defines_args = "--edtlib-Werror"
Anas Nashif50925412020-07-16 17:25:19 -04002142 else:
2143 ldflags = cflags = aflags = ""
Martí Bolívarfbd34dc2021-02-14 18:02:12 -08002144 gen_defines_args = ""
Anas Nashifce2b4182020-03-24 14:40:28 -04002145
Anas Nashif50925412020-07-16 17:25:19 -04002146 logger.debug("Running cmake on %s for %s" % (self.source_dir, self.platform.name))
Anas Nashifce2b4182020-03-24 14:40:28 -04002147 cmake_args = [
Anas Nashif50925412020-07-16 17:25:19 -04002148 f'-B{self.build_dir}',
2149 f'-S{self.source_dir}',
2150 f'-DEXTRA_CFLAGS="{cflags}"',
2151 f'-DEXTRA_AFLAGS="{aflags}',
2152 f'-DEXTRA_LDFLAGS="{ldflags}"',
Martí Bolívarfbd34dc2021-02-14 18:02:12 -08002153 f'-DEXTRA_GEN_DEFINES_ARGS={gen_defines_args}',
Anas Nashif50925412020-07-16 17:25:19 -04002154 f'-G{self.generator}'
Anas Nashifce2b4182020-03-24 14:40:28 -04002155 ]
2156
2157 if self.cmake_only:
2158 cmake_args.append("-DCMAKE_EXPORT_COMPILE_COMMANDS=1")
2159
2160 args = ["-D{}".format(a.replace('"', '')) for a in args]
2161 cmake_args.extend(args)
2162
2163 cmake_opts = ['-DBOARD={}'.format(self.platform.name)]
2164 cmake_args.extend(cmake_opts)
2165
2166
2167 logger.debug("Calling cmake with arguments: {}".format(cmake_args))
2168 cmake = shutil.which('cmake')
2169 cmd = [cmake] + cmake_args
2170 kwargs = dict()
2171
2172 if self.capture_output:
2173 kwargs['stdout'] = subprocess.PIPE
2174 # CMake sends the output of message() to stderr unless it's STATUS
2175 kwargs['stderr'] = subprocess.STDOUT
2176
2177 if self.cwd:
2178 kwargs['cwd'] = self.cwd
2179
2180 p = subprocess.Popen(cmd, **kwargs)
2181 out, _ = p.communicate()
2182
2183 if p.returncode == 0:
2184 filter_results = self.parse_generated()
2185 msg = "Finished building %s for %s" % (self.source_dir, self.platform.name)
2186 logger.debug(msg)
2187 results = {'msg': msg, 'filter': filter_results}
2188
2189 else:
Anas Nashiff04461e2020-06-29 10:07:02 -04002190 self.instance.status = "error"
Anas Nashifce2b4182020-03-24 14:40:28 -04002191 self.instance.reason = "Cmake build failure"
Piotr Golyzniak35a05202021-07-23 11:17:11 +02002192 self.instance.fill_results_by_status()
Anas Nashifce2b4182020-03-24 14:40:28 -04002193 logger.error("Cmake build failure: %s for %s" % (self.source_dir, self.platform.name))
2194 results = {"returncode": p.returncode}
2195
2196 if out:
2197 with open(os.path.join(self.build_dir, self.log), "a") as log:
2198 log_msg = out.decode(sys.getdefaultencoding())
2199 log.write(log_msg)
2200
2201 return results
2202
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01002203 @staticmethod
2204 def run_cmake_script(args=[]):
2205
2206 logger.debug("Running cmake script %s" % (args[0]))
2207
2208 cmake_args = ["-D{}".format(a.replace('"', '')) for a in args[1:]]
2209 cmake_args.extend(['-P', args[0]])
2210
2211 logger.debug("Calling cmake with arguments: {}".format(cmake_args))
2212 cmake = shutil.which('cmake')
Steven Huang1413c552021-03-14 01:54:40 -05002213 if not cmake:
2214 msg = "Unable to find `cmake` in path"
2215 logger.error(msg)
2216 raise Exception(msg)
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01002217 cmd = [cmake] + cmake_args
2218
2219 kwargs = dict()
2220 kwargs['stdout'] = subprocess.PIPE
2221 # CMake sends the output of message() to stderr unless it's STATUS
2222 kwargs['stderr'] = subprocess.STDOUT
2223
2224 p = subprocess.Popen(cmd, **kwargs)
2225 out, _ = p.communicate()
2226
Benedikt Schmidtb83681d2021-08-23 14:35:55 +02002227 # It might happen that the environment adds ANSI escape codes like \x1b[0m,
2228 # for instance if twister is executed from inside a makefile. In such a
2229 # scenario it is then necessary to remove them, as otherwise the JSON decoding
2230 # will fail.
2231 ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
2232 out = ansi_escape.sub('', out.decode())
2233
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01002234 if p.returncode == 0:
2235 msg = "Finished running %s" % (args[0])
2236 logger.debug(msg)
2237 results = {"returncode": p.returncode, "msg": msg, "stdout": out}
2238
2239 else:
2240 logger.error("Cmake script failure: %s" % (args[0]))
Torsten Rasmussenaf875992021-09-28 13:04:52 +02002241 results = {"returncode": p.returncode, "returnmsg": out}
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01002242
2243 return results
2244
Anas Nashifce2b4182020-03-24 14:40:28 -04002245
2246class FilterBuilder(CMake):
2247
2248 def __init__(self, testcase, platform, source_dir, build_dir):
2249 super().__init__(testcase, platform, source_dir, build_dir)
2250
Anas Nashifb18f3112020-12-07 11:40:19 -05002251 self.log = "config-twister.log"
Anas Nashifce2b4182020-03-24 14:40:28 -04002252
2253 def parse_generated(self):
2254
2255 if self.platform.name == "unit_testing":
2256 return {}
2257
2258 cmake_cache_path = os.path.join(self.build_dir, "CMakeCache.txt")
2259 defconfig_path = os.path.join(self.build_dir, "zephyr", ".config")
2260
2261 with open(defconfig_path, "r") as fp:
2262 defconfig = {}
2263 for line in fp.readlines():
2264 m = self.config_re.match(line)
2265 if not m:
2266 if line.strip() and not line.startswith("#"):
2267 sys.stderr.write("Unrecognized line %s\n" % line)
2268 continue
2269 defconfig[m.group(1)] = m.group(2).strip()
2270
2271 self.defconfig = defconfig
2272
2273 cmake_conf = {}
2274 try:
2275 cache = CMakeCache.from_file(cmake_cache_path)
2276 except FileNotFoundError:
2277 cache = {}
2278
2279 for k in iter(cache):
2280 cmake_conf[k.name] = k.value
2281
2282 self.cmake_cache = cmake_conf
2283
2284 filter_data = {
2285 "ARCH": self.platform.arch,
2286 "PLATFORM": self.platform.name
2287 }
2288 filter_data.update(os.environ)
2289 filter_data.update(self.defconfig)
2290 filter_data.update(self.cmake_cache)
2291
Martí Bolívar9c92baa2020-07-08 14:43:07 -07002292 edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle")
Anas Nashifce2b4182020-03-24 14:40:28 -04002293 if self.testcase and self.testcase.tc_filter:
2294 try:
Martí Bolívar9c92baa2020-07-08 14:43:07 -07002295 if os.path.exists(edt_pickle):
2296 with open(edt_pickle, 'rb') as f:
2297 edt = pickle.load(f)
Anas Nashifce2b4182020-03-24 14:40:28 -04002298 else:
2299 edt = None
2300 res = expr_parser.parse(self.testcase.tc_filter, filter_data, edt)
2301
2302 except (ValueError, SyntaxError) as se:
2303 sys.stderr.write(
2304 "Failed processing %s\n" % self.testcase.yamlfile)
2305 raise se
2306
2307 if not res:
2308 return {os.path.join(self.platform.name, self.testcase.name): True}
2309 else:
2310 return {os.path.join(self.platform.name, self.testcase.name): False}
2311 else:
2312 self.platform.filter_data = filter_data
2313 return filter_data
2314
2315
2316class ProjectBuilder(FilterBuilder):
2317
2318 def __init__(self, suite, instance, **kwargs):
2319 super().__init__(instance.testcase, instance.platform, instance.testcase.source_dir, instance.build_dir)
2320
2321 self.log = "build.log"
2322 self.instance = instance
2323 self.suite = suite
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002324 self.filtered_tests = 0
Anas Nashifce2b4182020-03-24 14:40:28 -04002325
2326 self.lsan = kwargs.get('lsan', False)
2327 self.asan = kwargs.get('asan', False)
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002328 self.ubsan = kwargs.get('ubsan', False)
Anas Nashifce2b4182020-03-24 14:40:28 -04002329 self.valgrind = kwargs.get('valgrind', False)
2330 self.extra_args = kwargs.get('extra_args', [])
2331 self.device_testing = kwargs.get('device_testing', False)
2332 self.cmake_only = kwargs.get('cmake_only', False)
2333 self.cleanup = kwargs.get('cleanup', False)
2334 self.coverage = kwargs.get('coverage', False)
2335 self.inline_logs = kwargs.get('inline_logs', False)
Anas Nashifce2b4182020-03-24 14:40:28 -04002336 self.generator = kwargs.get('generator', None)
2337 self.generator_cmd = kwargs.get('generator_cmd', None)
Anas Nashiff6462a32020-03-29 19:02:51 -04002338 self.verbose = kwargs.get('verbose', None)
Anas Nashif50925412020-07-16 17:25:19 -04002339 self.warnings_as_errors = kwargs.get('warnings_as_errors', True)
Anas Nashiff68146f2020-11-28 11:07:06 -05002340 self.overflow_as_errors = kwargs.get('overflow_as_errors', False)
Anas Nashifce2b4182020-03-24 14:40:28 -04002341
2342 @staticmethod
2343 def log_info(filename, inline_logs):
2344 filename = os.path.abspath(os.path.realpath(filename))
2345 if inline_logs:
2346 logger.info("{:-^100}".format(filename))
2347
2348 try:
2349 with open(filename) as fp:
2350 data = fp.read()
2351 except Exception as e:
2352 data = "Unable to read log data (%s)\n" % (str(e))
2353
2354 logger.error(data)
2355
2356 logger.info("{:-^100}".format(filename))
2357 else:
2358 logger.error("see: " + Fore.YELLOW + filename + Fore.RESET)
2359
2360 def log_info_file(self, inline_logs):
2361 build_dir = self.instance.build_dir
2362 h_log = "{}/handler.log".format(build_dir)
2363 b_log = "{}/build.log".format(build_dir)
2364 v_log = "{}/valgrind.log".format(build_dir)
2365 d_log = "{}/device.log".format(build_dir)
2366
2367 if os.path.exists(v_log) and "Valgrind" in self.instance.reason:
2368 self.log_info("{}".format(v_log), inline_logs)
2369 elif os.path.exists(h_log) and os.path.getsize(h_log) > 0:
2370 self.log_info("{}".format(h_log), inline_logs)
2371 elif os.path.exists(d_log) and os.path.getsize(d_log) > 0:
2372 self.log_info("{}".format(d_log), inline_logs)
2373 else:
2374 self.log_info("{}".format(b_log), inline_logs)
2375
2376 def setup_handler(self):
2377
2378 instance = self.instance
2379 args = []
2380
2381 # FIXME: Needs simplification
2382 if instance.platform.simulation == "qemu":
2383 instance.handler = QEMUHandler(instance, "qemu")
2384 args.append("QEMU_PIPE=%s" % instance.handler.get_fifo())
2385 instance.handler.call_make_run = True
2386 elif instance.testcase.type == "unit":
2387 instance.handler = BinaryHandler(instance, "unit")
2388 instance.handler.binary = os.path.join(instance.build_dir, "testbinary")
Anas Nashif051602f2020-04-28 14:27:46 -04002389 if self.coverage:
2390 args.append("COVERAGE=1")
Anas Nashifce2b4182020-03-24 14:40:28 -04002391 elif instance.platform.type == "native":
2392 handler = BinaryHandler(instance, "native")
2393
2394 handler.asan = self.asan
2395 handler.valgrind = self.valgrind
2396 handler.lsan = self.lsan
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002397 handler.ubsan = self.ubsan
Anas Nashifce2b4182020-03-24 14:40:28 -04002398 handler.coverage = self.coverage
2399
2400 handler.binary = os.path.join(instance.build_dir, "zephyr", "zephyr.exe")
2401 instance.handler = handler
Anas Nashifce2b4182020-03-24 14:40:28 -04002402 elif instance.platform.simulation == "renode":
2403 if find_executable("renode"):
2404 instance.handler = BinaryHandler(instance, "renode")
2405 instance.handler.pid_fn = os.path.join(instance.build_dir, "renode.pid")
2406 instance.handler.call_make_run = True
Martin Åbergc1077142020-10-19 18:21:52 +02002407 elif instance.platform.simulation == "tsim":
2408 instance.handler = BinaryHandler(instance, "tsim")
2409 instance.handler.call_make_run = True
Anas Nashifce2b4182020-03-24 14:40:28 -04002410 elif self.device_testing:
2411 instance.handler = DeviceHandler(instance, "device")
Andrei Emeltchenko10fa48d2021-01-12 09:56:32 +02002412 instance.handler.coverage = self.coverage
Eugeniy Paltsev9f2006c2020-10-08 22:31:19 +03002413 elif instance.platform.simulation == "nsim":
2414 if find_executable("nsimdrv"):
2415 instance.handler = BinaryHandler(instance, "nsim")
2416 instance.handler.call_make_run = True
Eugeniy Paltsev16032b672020-10-08 22:42:02 +03002417 elif instance.platform.simulation == "mdb-nsim":
2418 if find_executable("mdb"):
2419 instance.handler = BinaryHandler(instance, "nsim")
Jingru Wangb78d6742021-09-01 09:53:57 +08002420 instance.handler.call_make_run = True
Jaxson Han8af11d42021-02-24 10:23:46 +08002421 elif instance.platform.simulation == "armfvp":
2422 instance.handler = BinaryHandler(instance, "armfvp")
2423 instance.handler.call_make_run = True
Anas Nashifce2b4182020-03-24 14:40:28 -04002424
2425 if instance.handler:
2426 instance.handler.args = args
Anas Nashifb3669492020-03-24 22:33:50 -04002427 instance.handler.generator_cmd = self.generator_cmd
2428 instance.handler.generator = self.generator
Anas Nashifce2b4182020-03-24 14:40:28 -04002429
Anas Nashif531fe892020-09-11 13:56:33 -04002430 def process(self, pipeline, done, message, lock, results):
Anas Nashifce2b4182020-03-24 14:40:28 -04002431 op = message.get('op')
2432
2433 if not self.instance.handler:
2434 self.setup_handler()
2435
2436 # The build process, call cmake and build with configured generator
2437 if op == "cmake":
Anas Nashif531fe892020-09-11 13:56:33 -04002438 res = self.cmake()
Anas Nashiff04461e2020-06-29 10:07:02 -04002439 if self.instance.status in ["failed", "error"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04002440 pipeline.put({"op": "report", "test": self.instance})
2441 elif self.cmake_only:
Kumar Gala659b24b2020-10-09 09:51:02 -05002442 if self.instance.status is None:
2443 self.instance.status = "passed"
Anas Nashifce2b4182020-03-24 14:40:28 -04002444 pipeline.put({"op": "report", "test": self.instance})
2445 else:
Anas Nashif531fe892020-09-11 13:56:33 -04002446 if self.instance.name in res['filter'] and res['filter'][self.instance.name]:
Anas Nashifce2b4182020-03-24 14:40:28 -04002447 logger.debug("filtering %s" % self.instance.name)
2448 self.instance.status = "skipped"
2449 self.instance.reason = "filter"
Anas Nashif3b939da2020-11-24 13:21:27 -05002450 results.skipped_runtime += 1
Maciej Perkowskib2fa99c2020-05-21 14:45:29 +02002451 for case in self.instance.testcase.cases:
2452 self.instance.results.update({case: 'SKIP'})
Anas Nashifce2b4182020-03-24 14:40:28 -04002453 pipeline.put({"op": "report", "test": self.instance})
2454 else:
2455 pipeline.put({"op": "build", "test": self.instance})
2456
2457 elif op == "build":
2458 logger.debug("build test: %s" % self.instance.name)
Anas Nashif531fe892020-09-11 13:56:33 -04002459 res = self.build()
Anas Nashifce2b4182020-03-24 14:40:28 -04002460
Anas Nashif531fe892020-09-11 13:56:33 -04002461 if not res:
Anas Nashiff04461e2020-06-29 10:07:02 -04002462 self.instance.status = "error"
Anas Nashifce2b4182020-03-24 14:40:28 -04002463 self.instance.reason = "Build Failure"
2464 pipeline.put({"op": "report", "test": self.instance})
2465 else:
Anas Nashif531fe892020-09-11 13:56:33 -04002466 # Count skipped cases during build, for example
2467 # due to ram/rom overflow.
2468 inst = res.get("instance", None)
2469 if inst and inst.status == "skipped":
Anas Nashif3b939da2020-11-24 13:21:27 -05002470 results.skipped_runtime += 1
Anas Nashif531fe892020-09-11 13:56:33 -04002471
2472 if res.get('returncode', 1) > 0:
Anas Nashifce2b4182020-03-24 14:40:28 -04002473 pipeline.put({"op": "report", "test": self.instance})
2474 else:
Anas Nashif405f1b62020-07-27 12:27:13 -04002475 if self.instance.run and self.instance.handler:
Anas Nashifce2b4182020-03-24 14:40:28 -04002476 pipeline.put({"op": "run", "test": self.instance})
2477 else:
2478 pipeline.put({"op": "report", "test": self.instance})
2479 # Run the generated binary using one of the supported handlers
2480 elif op == "run":
2481 logger.debug("run test: %s" % self.instance.name)
2482 self.run()
2483 self.instance.status, _ = self.instance.handler.get_state()
Anas Nashif531fe892020-09-11 13:56:33 -04002484 logger.debug(f"run status: {self.instance.name} {self.instance.status}")
2485
2486 # to make it work with pickle
2487 self.instance.handler.thread = None
2488 self.instance.handler.suite = None
Anas Nashifce2b4182020-03-24 14:40:28 -04002489 pipeline.put({
2490 "op": "report",
2491 "test": self.instance,
Anas Nashifce2b4182020-03-24 14:40:28 -04002492 "status": self.instance.status,
Anas Nashif531fe892020-09-11 13:56:33 -04002493 "reason": self.instance.reason
2494 }
Anas Nashifce2b4182020-03-24 14:40:28 -04002495 )
2496
2497 # Report results and output progress to screen
2498 elif op == "report":
Anas Nashif531fe892020-09-11 13:56:33 -04002499 with lock:
2500 done.put(self.instance)
2501 self.report_out(results)
Anas Nashifce2b4182020-03-24 14:40:28 -04002502
2503 if self.cleanup and not self.coverage and self.instance.status == "passed":
2504 pipeline.put({
2505 "op": "cleanup",
2506 "test": self.instance
2507 })
2508
2509 elif op == "cleanup":
Kumar Gala1285c1f2020-09-24 13:21:07 -05002510 if self.device_testing:
2511 self.cleanup_device_testing_artifacts()
2512 else:
2513 self.cleanup_artifacts()
Anas Nashifce2b4182020-03-24 14:40:28 -04002514
Kumar Galaeb2e89a2020-10-20 15:44:15 -05002515 def cleanup_artifacts(self, additional_keep=[]):
Anas Nashifce2b4182020-03-24 14:40:28 -04002516 logger.debug("Cleaning up {}".format(self.instance.build_dir))
Anas Nashifdca317c2020-08-26 11:28:25 -04002517 allow = [
Anas Nashifce2b4182020-03-24 14:40:28 -04002518 'zephyr/.config',
2519 'handler.log',
2520 'build.log',
2521 'device.log',
Anas Nashif9ace63e2020-04-28 07:14:43 -04002522 'recording.csv',
Anas Nashifce2b4182020-03-24 14:40:28 -04002523 ]
Kumar Gala1285c1f2020-09-24 13:21:07 -05002524
2525 allow += additional_keep
2526
Anas Nashifdca317c2020-08-26 11:28:25 -04002527 allow = [os.path.join(self.instance.build_dir, file) for file in allow]
Anas Nashifce2b4182020-03-24 14:40:28 -04002528
2529 for dirpath, dirnames, filenames in os.walk(self.instance.build_dir, topdown=False):
2530 for name in filenames:
2531 path = os.path.join(dirpath, name)
Anas Nashifdca317c2020-08-26 11:28:25 -04002532 if path not in allow:
Anas Nashifce2b4182020-03-24 14:40:28 -04002533 os.remove(path)
2534 # Remove empty directories and symbolic links to directories
2535 for dir in dirnames:
2536 path = os.path.join(dirpath, dir)
2537 if os.path.islink(path):
2538 os.remove(path)
2539 elif not os.listdir(path):
2540 os.rmdir(path)
2541
Kumar Gala1285c1f2020-09-24 13:21:07 -05002542 def cleanup_device_testing_artifacts(self):
2543 logger.debug("Cleaning up for Device Testing {}".format(self.instance.build_dir))
2544
Kumar Galab2c07e42020-10-01 07:25:50 -05002545 sanitizelist = [
Kumar Gala1285c1f2020-09-24 13:21:07 -05002546 'CMakeCache.txt',
2547 'zephyr/runners.yaml',
Kumar Galab2c07e42020-10-01 07:25:50 -05002548 ]
2549 keep = [
Kumar Gala1285c1f2020-09-24 13:21:07 -05002550 'zephyr/zephyr.hex',
2551 'zephyr/zephyr.bin',
2552 'zephyr/zephyr.elf',
2553 ]
2554
Kumar Galab2c07e42020-10-01 07:25:50 -05002555 keep += sanitizelist
2556
Kumar Gala1285c1f2020-09-24 13:21:07 -05002557 self.cleanup_artifacts(keep)
2558
Kumar Galab2c07e42020-10-01 07:25:50 -05002559 # sanitize paths so files are relocatable
2560 for file in sanitizelist:
2561 file = os.path.join(self.instance.build_dir, file)
2562
2563 with open(file, "rt") as fin:
2564 data = fin.read()
2565 data = data.replace(canonical_zephyr_base+"/", "")
2566
2567 with open(file, "wt") as fin:
2568 fin.write(data)
2569
Anas Nashif531fe892020-09-11 13:56:33 -04002570 def report_out(self, results):
Anas Nashif3b939da2020-11-24 13:21:27 -05002571 total_to_do = results.total - results.skipped_configs
Anas Nashif531fe892020-09-11 13:56:33 -04002572 total_tests_width = len(str(total_to_do))
2573 results.done += 1
Anas Nashifce2b4182020-03-24 14:40:28 -04002574 instance = self.instance
2575
Maciej Perkowskif050a992021-02-24 13:43:05 +01002576 if instance.status in ["error", "failed", "timeout", "flash_error"]:
Anas Nashifdc43c292020-07-09 09:46:45 -04002577 if instance.status == "error":
Anas Nashif531fe892020-09-11 13:56:33 -04002578 results.error += 1
2579 results.failed += 1
Anas Nashiff6462a32020-03-29 19:02:51 -04002580 if self.verbose:
Anas Nashifce2b4182020-03-24 14:40:28 -04002581 status = Fore.RED + "FAILED " + Fore.RESET + instance.reason
2582 else:
2583 print("")
2584 logger.error(
2585 "{:<25} {:<50} {}FAILED{}: {}".format(
2586 instance.platform.name,
2587 instance.testcase.name,
2588 Fore.RED,
2589 Fore.RESET,
2590 instance.reason))
Anas Nashiff6462a32020-03-29 19:02:51 -04002591 if not self.verbose:
Anas Nashifce2b4182020-03-24 14:40:28 -04002592 self.log_info_file(self.inline_logs)
2593 elif instance.status == "skipped":
Anas Nashifce2b4182020-03-24 14:40:28 -04002594 status = Fore.YELLOW + "SKIPPED" + Fore.RESET
Anas Nashif869ca052020-07-07 14:29:07 -04002595 elif instance.status == "passed":
Anas Nashifce2b4182020-03-24 14:40:28 -04002596 status = Fore.GREEN + "PASSED" + Fore.RESET
Anas Nashif869ca052020-07-07 14:29:07 -04002597 else:
2598 logger.debug(f"Unknown status = {instance.status}")
2599 status = Fore.YELLOW + "UNKNOWN" + Fore.RESET
Anas Nashifce2b4182020-03-24 14:40:28 -04002600
Anas Nashiff6462a32020-03-29 19:02:51 -04002601 if self.verbose:
Anas Nashifce2b4182020-03-24 14:40:28 -04002602 if self.cmake_only:
2603 more_info = "cmake"
2604 elif instance.status == "skipped":
2605 more_info = instance.reason
2606 else:
2607 if instance.handler and instance.run:
2608 more_info = instance.handler.type_str
2609 htime = instance.handler.duration
2610 if htime:
2611 more_info += " {:.3f}s".format(htime)
2612 else:
2613 more_info = "build"
2614
2615 logger.info("{:>{}}/{} {:<25} {:<50} {} ({})".format(
Anas Nashif531fe892020-09-11 13:56:33 -04002616 results.done, total_tests_width, total_to_do, instance.platform.name,
Anas Nashifce2b4182020-03-24 14:40:28 -04002617 instance.testcase.name, status, more_info))
2618
Anas Nashiff04461e2020-06-29 10:07:02 -04002619 if instance.status in ["error", "failed", "timeout"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04002620 self.log_info_file(self.inline_logs)
2621 else:
Maciej Perkowski2ec5ec22020-09-28 12:37:55 +02002622 completed_perc = 0
Anas Nashif531fe892020-09-11 13:56:33 -04002623 if total_to_do > 0:
2624 completed_perc = int((float(results.done) / total_to_do) * 100)
Maciej Perkowski2ec5ec22020-09-28 12:37:55 +02002625
Anas Nashif3b939da2020-11-24 13:21:27 -05002626 skipped = results.skipped_configs + results.skipped_runtime
Anas Nashifce2b4182020-03-24 14:40:28 -04002627 sys.stdout.write("\rINFO - Total complete: %s%4d/%4d%s %2d%% skipped: %s%4d%s, failed: %s%4d%s" % (
2628 Fore.GREEN,
Anas Nashif531fe892020-09-11 13:56:33 -04002629 results.done,
2630 total_to_do,
Anas Nashifce2b4182020-03-24 14:40:28 -04002631 Fore.RESET,
Maciej Perkowski2ec5ec22020-09-28 12:37:55 +02002632 completed_perc,
Anas Nashif531fe892020-09-11 13:56:33 -04002633 Fore.YELLOW if skipped > 0 else Fore.RESET,
2634 skipped,
Anas Nashifce2b4182020-03-24 14:40:28 -04002635 Fore.RESET,
Anas Nashif531fe892020-09-11 13:56:33 -04002636 Fore.RED if results.failed > 0 else Fore.RESET,
2637 results.failed,
Anas Nashifce2b4182020-03-24 14:40:28 -04002638 Fore.RESET
2639 )
2640 )
2641 sys.stdout.flush()
2642
2643 def cmake(self):
2644
2645 instance = self.instance
2646 args = self.testcase.extra_args[:]
2647 args += self.extra_args
2648
2649 if instance.handler:
2650 args += instance.handler.args
2651
2652 # merge overlay files into one variable
2653 def extract_overlays(args):
2654 re_overlay = re.compile('OVERLAY_CONFIG=(.*)')
2655 other_args = []
2656 overlays = []
2657 for arg in args:
2658 match = re_overlay.search(arg)
2659 if match:
2660 overlays.append(match.group(1).strip('\'"'))
2661 else:
2662 other_args.append(arg)
2663
2664 args[:] = other_args
2665 return overlays
2666
2667 overlays = extract_overlays(args)
2668
Torsten Rasmussenca08cc02020-11-25 21:56:46 +01002669 if os.path.exists(os.path.join(instance.build_dir,
Anas Nashifb18f3112020-12-07 11:40:19 -05002670 "twister", "testcase_extra.conf")):
Anas Nashifce2b4182020-03-24 14:40:28 -04002671 overlays.append(os.path.join(instance.build_dir,
Anas Nashifb18f3112020-12-07 11:40:19 -05002672 "twister", "testcase_extra.conf"))
Anas Nashifce2b4182020-03-24 14:40:28 -04002673
2674 if overlays:
2675 args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays)))
2676
Anas Nashif531fe892020-09-11 13:56:33 -04002677 res = self.run_cmake(args)
2678 return res
Anas Nashifce2b4182020-03-24 14:40:28 -04002679
2680 def build(self):
Anas Nashif531fe892020-09-11 13:56:33 -04002681 res = self.run_build(['--build', self.build_dir])
2682 return res
Anas Nashifce2b4182020-03-24 14:40:28 -04002683
2684 def run(self):
2685
2686 instance = self.instance
2687
Anas Nashif405f1b62020-07-27 12:27:13 -04002688 if instance.handler:
2689 if instance.handler.type_str == "device":
2690 instance.handler.suite = self.suite
Anas Nashifce2b4182020-03-24 14:40:28 -04002691
Anas Nashif405f1b62020-07-27 12:27:13 -04002692 instance.handler.handle()
Anas Nashifce2b4182020-03-24 14:40:28 -04002693
2694 sys.stdout.flush()
2695
Anas Nashifaff616d2020-04-17 21:24:57 -04002696class TestSuite(DisablePyTestCollectionMixin):
Anas Nashifce2b4182020-03-24 14:40:28 -04002697 config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
2698 dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
2699
2700 tc_schema = scl.yaml_load(
2701 os.path.join(ZEPHYR_BASE,
Anas Nashifc24bf6f2020-12-07 11:18:07 -05002702 "scripts", "schemas", "twister", "testcase-schema.yaml"))
Maciej Perkowski9c6dfce2021-03-11 13:18:33 +01002703 quarantine_schema = scl.yaml_load(
2704 os.path.join(ZEPHYR_BASE,
2705 "scripts", "schemas", "twister", "quarantine-schema.yaml"))
Anas Nashifce2b4182020-03-24 14:40:28 -04002706
2707 testcase_valid_keys = {"tags": {"type": "set", "required": False},
2708 "type": {"type": "str", "default": "integration"},
2709 "extra_args": {"type": "list"},
2710 "extra_configs": {"type": "list"},
2711 "build_only": {"type": "bool", "default": False},
2712 "build_on_all": {"type": "bool", "default": False},
2713 "skip": {"type": "bool", "default": False},
2714 "slow": {"type": "bool", "default": False},
2715 "timeout": {"type": "int", "default": 60},
2716 "min_ram": {"type": "int", "default": 8},
2717 "depends_on": {"type": "set"},
2718 "min_flash": {"type": "int", "default": 32},
Anas Nashifdca317c2020-08-26 11:28:25 -04002719 "arch_allow": {"type": "set"},
Anas Nashifce2b4182020-03-24 14:40:28 -04002720 "arch_exclude": {"type": "set"},
2721 "extra_sections": {"type": "list", "default": []},
Anas Nashif1636c312020-05-28 08:02:54 -04002722 "integration_platforms": {"type": "list", "default": []},
Anas Nashifce2b4182020-03-24 14:40:28 -04002723 "platform_exclude": {"type": "set"},
Anas Nashifdca317c2020-08-26 11:28:25 -04002724 "platform_allow": {"type": "set"},
Anas Nashifce2b4182020-03-24 14:40:28 -04002725 "toolchain_exclude": {"type": "set"},
Anas Nashifdca317c2020-08-26 11:28:25 -04002726 "toolchain_allow": {"type": "set"},
Anas Nashifce2b4182020-03-24 14:40:28 -04002727 "filter": {"type": "str"},
2728 "harness": {"type": "str"},
2729 "harness_config": {"type": "map", "default": {}}
2730 }
2731
Anas Nashif94f68262020-12-07 11:21:04 -05002732 RELEASE_DATA = os.path.join(ZEPHYR_BASE, "scripts", "release",
2733 "twister_last_release.csv")
Anas Nashifce2b4182020-03-24 14:40:28 -04002734
Aastha Grovera0ae5342020-05-13 13:34:00 -07002735 SAMPLE_FILENAME = 'sample.yaml'
2736 TESTCASE_FILENAME = 'testcase.yaml'
2737
Anas Nashifaff616d2020-04-17 21:24:57 -04002738 def __init__(self, board_root_list=[], testcase_roots=[], outdir=None):
Anas Nashifce2b4182020-03-24 14:40:28 -04002739
2740 self.roots = testcase_roots
2741 if not isinstance(board_root_list, list):
2742 self.board_roots = [board_root_list]
2743 else:
2744 self.board_roots = board_root_list
2745
2746 # Testsuite Options
2747 self.coverage_platform = []
2748 self.build_only = False
2749 self.cmake_only = False
2750 self.cleanup = False
2751 self.enable_slow = False
2752 self.device_testing = False
Anas Nashifce8c12e2020-05-21 09:11:40 -04002753 self.fixtures = []
Anas Nashifce2b4182020-03-24 14:40:28 -04002754 self.enable_coverage = False
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002755 self.enable_ubsan = False
Anas Nashifce2b4182020-03-24 14:40:28 -04002756 self.enable_lsan = False
2757 self.enable_asan = False
2758 self.enable_valgrind = False
2759 self.extra_args = []
2760 self.inline_logs = False
2761 self.enable_sizes_report = False
2762 self.west_flash = None
2763 self.west_runner = None
2764 self.generator = None
2765 self.generator_cmd = None
Anas Nashif50925412020-07-16 17:25:19 -04002766 self.warnings_as_errors = True
Anas Nashiff68146f2020-11-28 11:07:06 -05002767 self.overflow_as_errors = False
Maciej Perkowski9c6dfce2021-03-11 13:18:33 +01002768 self.quarantine_verify = False
Anas Nashifce2b4182020-03-24 14:40:28 -04002769
2770 # Keep track of which test cases we've filtered out and why
2771 self.testcases = {}
Maciej Perkowski9c6dfce2021-03-11 13:18:33 +01002772 self.quarantine = {}
Anas Nashifce2b4182020-03-24 14:40:28 -04002773 self.platforms = []
2774 self.selected_platforms = []
Maciej Perkowskib6a69aa2021-02-04 16:05:21 +01002775 self.filtered_platforms = []
Anas Nashifce2b4182020-03-24 14:40:28 -04002776 self.default_platforms = []
2777 self.outdir = os.path.abspath(outdir)
Anas Nashifaff616d2020-04-17 21:24:57 -04002778 self.discards = {}
Anas Nashifce2b4182020-03-24 14:40:28 -04002779 self.load_errors = 0
2780 self.instances = dict()
2781
Anas Nashifce2b4182020-03-24 14:40:28 -04002782 self.total_platforms = 0
2783 self.start_time = 0
2784 self.duration = 0
2785 self.warnings = 0
Anas Nashifce2b4182020-03-24 14:40:28 -04002786
2787 # hardcoded for now
Anas Nashif8305d1b2020-11-26 11:55:02 -05002788 self.duts = []
Anas Nashifce2b4182020-03-24 14:40:28 -04002789
Anas Nashif1636c312020-05-28 08:02:54 -04002790 # run integration tests only
2791 self.integration = False
2792
Anas Nashif531fe892020-09-11 13:56:33 -04002793 self.pipeline = None
Maciej Perkowski060e00d2020-10-13 18:59:37 +02002794 self.version = "NA"
2795
2796 def check_zephyr_version(self):
2797 try:
Jingru Wangb9c953c2020-12-21 10:45:27 +08002798 subproc = subprocess.run(["git", "describe", "--abbrev=12"],
Maciej Perkowski060e00d2020-10-13 18:59:37 +02002799 stdout=subprocess.PIPE,
2800 universal_newlines=True,
2801 cwd=ZEPHYR_BASE)
2802 if subproc.returncode == 0:
2803 self.version = subproc.stdout.strip()
2804 logger.info(f"Zephyr version: {self.version}")
2805 except OSError:
2806 logger.info("Cannot read zephyr version.")
2807
Anas Nashifbb280352020-05-07 12:02:48 -04002808 def get_platform_instances(self, platform):
Piotr Golyzniak8b773482021-11-02 16:13:26 +01002809 filtered_dict = {k:v for k,v in self.instances.items() if k.startswith(platform + os.sep)}
Anas Nashifbb280352020-05-07 12:02:48 -04002810 return filtered_dict
2811
Anas Nashifce2b4182020-03-24 14:40:28 -04002812 def config(self):
2813 logger.info("coverage platform: {}".format(self.coverage_platform))
2814
2815 # Debug Functions
2816 @staticmethod
2817 def info(what):
2818 sys.stdout.write(what + "\n")
2819 sys.stdout.flush()
2820
Anas Nashif531fe892020-09-11 13:56:33 -04002821 def update_counting(self, results=None, initial=False):
Anas Nashif3b939da2020-11-24 13:21:27 -05002822 results.skipped_configs = 0
2823 results.skipped_cases = 0
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002824 for instance in self.instances.values():
Anas Nashif531fe892020-09-11 13:56:33 -04002825 if initial:
2826 results.cases += len(instance.testcase.cases)
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002827 if instance.status == 'skipped':
Anas Nashif3b939da2020-11-24 13:21:27 -05002828 results.skipped_configs += 1
Anas Nashif531fe892020-09-11 13:56:33 -04002829 results.skipped_cases += len(instance.testcase.cases)
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002830 elif instance.status == "passed":
Anas Nashif531fe892020-09-11 13:56:33 -04002831 results.passed += 1
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002832 for res in instance.results.values():
2833 if res == 'SKIP':
Anas Nashif531fe892020-09-11 13:56:33 -04002834 results.skipped_cases += 1
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002835
Anas Nashifce2b4182020-03-24 14:40:28 -04002836 def compare_metrics(self, filename):
2837 # name, datatype, lower results better
2838 interesting_metrics = [("ram_size", int, True),
2839 ("rom_size", int, True)]
2840
2841 if not os.path.exists(filename):
Anas Nashifad44bed2020-08-24 18:59:01 -04002842 logger.error("Cannot compare metrics, %s not found" % filename)
Anas Nashifce2b4182020-03-24 14:40:28 -04002843 return []
2844
2845 results = []
2846 saved_metrics = {}
2847 with open(filename) as fp:
2848 cr = csv.DictReader(fp)
2849 for row in cr:
2850 d = {}
2851 for m, _, _ in interesting_metrics:
2852 d[m] = row[m]
2853 saved_metrics[(row["test"], row["platform"])] = d
2854
2855 for instance in self.instances.values():
2856 mkey = (instance.testcase.name, instance.platform.name)
2857 if mkey not in saved_metrics:
2858 continue
2859 sm = saved_metrics[mkey]
2860 for metric, mtype, lower_better in interesting_metrics:
2861 if metric not in instance.metrics:
2862 continue
2863 if sm[metric] == "":
2864 continue
2865 delta = instance.metrics.get(metric, 0) - mtype(sm[metric])
2866 if delta == 0:
2867 continue
2868 results.append((instance, metric, instance.metrics.get(metric, 0), delta,
2869 lower_better))
2870 return results
2871
Anas Nashifad44bed2020-08-24 18:59:01 -04002872 def footprint_reports(self, report, show_footprint, all_deltas,
2873 footprint_threshold, last_metrics):
Anas Nashifce2b4182020-03-24 14:40:28 -04002874 if not report:
2875 return
2876
Anas Nashifad44bed2020-08-24 18:59:01 -04002877 logger.debug("running footprint_reports")
Anas Nashifce2b4182020-03-24 14:40:28 -04002878 deltas = self.compare_metrics(report)
2879 warnings = 0
2880 if deltas and show_footprint:
2881 for i, metric, value, delta, lower_better in deltas:
2882 if not all_deltas and ((delta < 0 and lower_better) or
2883 (delta > 0 and not lower_better)):
2884 continue
2885
Anas Nashifad44bed2020-08-24 18:59:01 -04002886 percentage = 0
2887 if value > delta:
2888 percentage = (float(delta) / float(value - delta))
2889
2890 if not all_deltas and (percentage < (footprint_threshold / 100.0)):
Anas Nashifce2b4182020-03-24 14:40:28 -04002891 continue
2892
2893 logger.info("{:<25} {:<60} {}{}{}: {} {:<+4}, is now {:6} {:+.2%}".format(
2894 i.platform.name, i.testcase.name, Fore.YELLOW,
2895 "INFO" if all_deltas else "WARNING", Fore.RESET,
2896 metric, delta, value, percentage))
2897 warnings += 1
2898
2899 if warnings:
2900 logger.warning("Deltas based on metrics from last %s" %
2901 ("release" if not last_metrics else "run"))
2902
Anas Nashif531fe892020-09-11 13:56:33 -04002903 def summary(self, results, unrecognized_sections):
Anas Nashifce2b4182020-03-24 14:40:28 -04002904 failed = 0
Anas Nashif4258d8d2020-05-08 08:40:27 -04002905 run = 0
Anas Nashifce2b4182020-03-24 14:40:28 -04002906 for instance in self.instances.values():
2907 if instance.status == "failed":
2908 failed += 1
2909 elif instance.metrics.get("unrecognized") and not unrecognized_sections:
2910 logger.error("%sFAILED%s: %s has unrecognized binary sections: %s" %
2911 (Fore.RED, Fore.RESET, instance.name,
2912 str(instance.metrics.get("unrecognized", []))))
2913 failed += 1
2914
Anas Nashifad44bed2020-08-24 18:59:01 -04002915 if instance.metrics.get('handler_time', None):
Anas Nashif4258d8d2020-05-08 08:40:27 -04002916 run += 1
2917
Anas Nashif3b939da2020-11-24 13:21:27 -05002918 if results.total and results.total != results.skipped_configs:
2919 pass_rate = (float(results.passed) / float(results.total - results.skipped_configs))
Anas Nashifce2b4182020-03-24 14:40:28 -04002920 else:
2921 pass_rate = 0
2922
2923 logger.info(
Anas Nashif3b939da2020-11-24 13:21:27 -05002924 "{}{} of {}{} test configurations passed ({:.2%}), {}{}{} failed, {} skipped with {}{}{} warnings in {:.2f} seconds".format(
Anas Nashifce2b4182020-03-24 14:40:28 -04002925 Fore.RED if failed else Fore.GREEN,
Anas Nashif531fe892020-09-11 13:56:33 -04002926 results.passed,
Anas Nashif3b939da2020-11-24 13:21:27 -05002927 results.total - results.skipped_configs,
Anas Nashifce2b4182020-03-24 14:40:28 -04002928 Fore.RESET,
2929 pass_rate,
Anas Nashif531fe892020-09-11 13:56:33 -04002930 Fore.RED if results.failed else Fore.RESET,
2931 results.failed,
Anas Nashifce2b4182020-03-24 14:40:28 -04002932 Fore.RESET,
Anas Nashif3b939da2020-11-24 13:21:27 -05002933 results.skipped_configs,
Anas Nashifce2b4182020-03-24 14:40:28 -04002934 Fore.YELLOW if self.warnings else Fore.RESET,
2935 self.warnings,
2936 Fore.RESET,
2937 self.duration))
2938
2939 self.total_platforms = len(self.platforms)
Anas Nashifbb427952020-11-24 08:39:42 -05002940 # if we are only building, do not report about tests being executed.
2941 if self.platforms and not self.build_only:
Anas Nashif531fe892020-09-11 13:56:33 -04002942 logger.info("In total {} test cases were executed, {} skipped on {} out of total {} platforms ({:02.2f}%)".format(
2943 results.cases - results.skipped_cases,
2944 results.skipped_cases,
Maciej Perkowskib6a69aa2021-02-04 16:05:21 +01002945 len(self.filtered_platforms),
Anas Nashifce2b4182020-03-24 14:40:28 -04002946 self.total_platforms,
Maciej Perkowskib6a69aa2021-02-04 16:05:21 +01002947 (100 * len(self.filtered_platforms) / len(self.platforms))
Anas Nashifce2b4182020-03-24 14:40:28 -04002948 ))
2949
Anas Nashif3b939da2020-11-24 13:21:27 -05002950 logger.info(f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms, \
2951{Fore.RED}{results.total - run - results.skipped_configs}{Fore.RESET} test configurations were only built.")
Anas Nashif4258d8d2020-05-08 08:40:27 -04002952
Anas Nashifec479122021-01-25 08:13:27 -05002953 def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report):
Anas Nashifce2b4182020-03-24 14:40:28 -04002954 if not self.instances:
2955 return
2956
Anas Nashifa5d16ab2020-12-10 11:45:57 -05002957 logger.info("Saving reports...")
Anas Nashifce2b4182020-03-24 14:40:28 -04002958 if name:
2959 report_name = name
2960 else:
Anas Nashifb18f3112020-12-07 11:40:19 -05002961 report_name = "twister"
Anas Nashifce2b4182020-03-24 14:40:28 -04002962
2963 if report_dir:
2964 os.makedirs(report_dir, exist_ok=True)
2965 filename = os.path.join(report_dir, report_name)
2966 outdir = report_dir
2967 else:
2968 filename = os.path.join(self.outdir, report_name)
2969 outdir = self.outdir
2970
Anas Nashif6915adf2020-04-22 09:39:42 -04002971 if suffix:
2972 filename = "{}_{}".format(filename, suffix)
2973
Anas Nashifce2b4182020-03-24 14:40:28 -04002974 if not no_update:
Maciej Perkowski060e00d2020-10-13 18:59:37 +02002975 self.xunit_report(filename + ".xml", full_report=False,
2976 append=only_failed, version=self.version)
2977 self.xunit_report(filename + "_report.xml", full_report=True,
2978 append=only_failed, version=self.version)
Anas Nashifce2b4182020-03-24 14:40:28 -04002979 self.csv_report(filename + ".csv")
Anas Nashifec479122021-01-25 08:13:27 -05002980
2981 if json_report:
2982 self.json_report(filename + ".json", append=only_failed, version=self.version)
Anas Nashif90415502020-04-11 22:15:04 -04002983
Anas Nashifa5d16ab2020-12-10 11:45:57 -05002984 if platform_reports:
2985 self.target_report(outdir, suffix, append=only_failed)
Anas Nashifce2b4182020-03-24 14:40:28 -04002986 if self.discards:
2987 self.discard_report(filename + "_discard.csv")
2988
2989 if release:
2990 self.csv_report(self.RELEASE_DATA)
2991
2992 def add_configurations(self):
2993
2994 for board_root in self.board_roots:
2995 board_root = os.path.abspath(board_root)
2996
2997 logger.debug("Reading platform configuration files under %s..." %
2998 board_root)
2999
3000 for file in glob.glob(os.path.join(board_root, "*", "*", "*.yaml")):
Anas Nashifce2b4182020-03-24 14:40:28 -04003001 try:
3002 platform = Platform()
3003 platform.load(file)
Anas Nashif61c4a512020-08-13 07:46:45 -04003004 if platform.name in [p.name for p in self.platforms]:
3005 logger.error(f"Duplicate platform {platform.name} in {file}")
3006 raise Exception(f"Duplicate platform identifier {platform.name} found")
Anas Nashifb18f3112020-12-07 11:40:19 -05003007 if platform.twister:
Anas Nashifce2b4182020-03-24 14:40:28 -04003008 self.platforms.append(platform)
3009 if platform.default:
3010 self.default_platforms.append(platform.name)
3011
3012 except RuntimeError as e:
3013 logger.error("E: %s: can't load: %s" % (file, e))
3014 self.load_errors += 1
3015
3016 def get_all_tests(self):
3017 tests = []
3018 for _, tc in self.testcases.items():
3019 for case in tc.cases:
3020 tests.append(case)
3021
3022 return tests
3023
3024 @staticmethod
3025 def get_toolchain():
Torsten Rasmussena0889702021-02-05 09:55:41 +01003026 toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake')
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01003027 result = CMake.run_cmake_script([toolchain_script, "FORMAT=json"])
3028
Anas Nashifce2b4182020-03-24 14:40:28 -04003029 try:
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01003030 if result['returncode']:
Torsten Rasmussenaf875992021-09-28 13:04:52 +02003031 raise TwisterRuntimeError(f"E: {result['returnmsg']}")
Anas Nashifce2b4182020-03-24 14:40:28 -04003032 except Exception as e:
3033 print(str(e))
3034 sys.exit(2)
Torsten Rasmussend162e9e2021-02-04 15:55:23 +01003035 toolchain = json.loads(result['stdout'])['ZEPHYR_TOOLCHAIN_VARIANT']
3036 logger.info(f"Using '{toolchain}' toolchain.")
Anas Nashifce2b4182020-03-24 14:40:28 -04003037
3038 return toolchain
3039
3040 def add_testcases(self, testcase_filter=[]):
3041 for root in self.roots:
3042 root = os.path.abspath(root)
3043
3044 logger.debug("Reading test case configuration files under %s..." % root)
3045
Jorgen Kvalvaag36d96d22021-04-06 14:57:28 +02003046 for dirpath, _, filenames in os.walk(root, topdown=True):
Aastha Grovera0ae5342020-05-13 13:34:00 -07003047 if self.SAMPLE_FILENAME in filenames:
3048 filename = self.SAMPLE_FILENAME
3049 elif self.TESTCASE_FILENAME in filenames:
3050 filename = self.TESTCASE_FILENAME
Anas Nashifce2b4182020-03-24 14:40:28 -04003051 else:
3052 continue
3053
3054 logger.debug("Found possible test case in " + dirpath)
3055
Anas Nashifce2b4182020-03-24 14:40:28 -04003056 tc_path = os.path.join(dirpath, filename)
3057
3058 try:
Anas Nashif45943702020-12-11 17:55:15 -05003059 parsed_data = TwisterConfigParser(tc_path, self.tc_schema)
Anas Nashifce2b4182020-03-24 14:40:28 -04003060 parsed_data.load()
3061
3062 tc_path = os.path.dirname(tc_path)
3063 workdir = os.path.relpath(tc_path, root)
3064
3065 for name in parsed_data.tests.keys():
Anas Nashifaff616d2020-04-17 21:24:57 -04003066 tc = TestCase(root, workdir, name)
Anas Nashifce2b4182020-03-24 14:40:28 -04003067
3068 tc_dict = parsed_data.get_test(name, self.testcase_valid_keys)
3069
3070 tc.source_dir = tc_path
3071 tc.yamlfile = tc_path
3072
Anas Nashifce2b4182020-03-24 14:40:28 -04003073 tc.type = tc_dict["type"]
3074 tc.tags = tc_dict["tags"]
3075 tc.extra_args = tc_dict["extra_args"]
3076 tc.extra_configs = tc_dict["extra_configs"]
Anas Nashifdca317c2020-08-26 11:28:25 -04003077 tc.arch_allow = tc_dict["arch_allow"]
Anas Nashifce2b4182020-03-24 14:40:28 -04003078 tc.arch_exclude = tc_dict["arch_exclude"]
3079 tc.skip = tc_dict["skip"]
3080 tc.platform_exclude = tc_dict["platform_exclude"]
Anas Nashifdca317c2020-08-26 11:28:25 -04003081 tc.platform_allow = tc_dict["platform_allow"]
Anas Nashifce2b4182020-03-24 14:40:28 -04003082 tc.toolchain_exclude = tc_dict["toolchain_exclude"]
Anas Nashifdca317c2020-08-26 11:28:25 -04003083 tc.toolchain_allow = tc_dict["toolchain_allow"]
Anas Nashifce2b4182020-03-24 14:40:28 -04003084 tc.tc_filter = tc_dict["filter"]
3085 tc.timeout = tc_dict["timeout"]
3086 tc.harness = tc_dict["harness"]
3087 tc.harness_config = tc_dict["harness_config"]
Anas Nashif43275c82020-05-04 18:22:16 -04003088 if tc.harness == 'console' and not tc.harness_config:
3089 raise Exception('Harness config error: console harness defined without a configuration.')
Anas Nashifce2b4182020-03-24 14:40:28 -04003090 tc.build_only = tc_dict["build_only"]
3091 tc.build_on_all = tc_dict["build_on_all"]
3092 tc.slow = tc_dict["slow"]
3093 tc.min_ram = tc_dict["min_ram"]
3094 tc.depends_on = tc_dict["depends_on"]
3095 tc.min_flash = tc_dict["min_flash"]
3096 tc.extra_sections = tc_dict["extra_sections"]
Anas Nashif1636c312020-05-28 08:02:54 -04003097 tc.integration_platforms = tc_dict["integration_platforms"]
Anas Nashifce2b4182020-03-24 14:40:28 -04003098
3099 tc.parse_subcases(tc_path)
3100
3101 if testcase_filter:
3102 if tc.name and tc.name in testcase_filter:
3103 self.testcases[tc.name] = tc
3104 else:
3105 self.testcases[tc.name] = tc
3106
3107 except Exception as e:
3108 logger.error("%s: can't load (skipping): %s" % (tc_path, e))
3109 self.load_errors += 1
Anas Nashiffe07d572020-11-20 12:13:47 -05003110 return len(self.testcases)
Anas Nashifce2b4182020-03-24 14:40:28 -04003111
3112 def get_platform(self, name):
3113 selected_platform = None
3114 for platform in self.platforms:
3115 if platform.name == name:
3116 selected_platform = platform
3117 break
3118 return selected_platform
3119
Maciej Perkowski9c6dfce2021-03-11 13:18:33 +01003120 def load_quarantine(self, file):
3121 """
3122 Loads quarantine list from the given yaml file. Creates a dictionary
3123 of all tests configurations (platform + scenario: comment) that shall be
3124 skipped due to quarantine
3125 """
3126
3127 # Load yaml into quarantine_yaml
3128 quarantine_yaml = scl.yaml_load_verify(file, self.quarantine_schema)
3129
3130 # Create quarantine_list with a product of the listed
3131 # platforms and scenarios for each entry in quarantine yaml
3132 quarantine_list = []
3133 for quar_dict in quarantine_yaml:
3134 if quar_dict['platforms'][0] == "all":
3135 plat = [p.name for p in self.platforms]
3136 else:
3137 plat = quar_dict['platforms']
3138 comment = quar_dict.get('comment', "NA")
3139 quarantine_list.append([{".".join([p, s]): comment}
3140 for p in plat for s in quar_dict['scenarios']])
3141
3142 # Flatten the quarantine_list
3143 quarantine_list = [it for sublist in quarantine_list for it in sublist]
3144 # Change quarantine_list into a dictionary
3145 for d in quarantine_list:
3146 self.quarantine.update(d)
3147
Anas Nashif82a6a462020-11-02 10:47:36 -05003148 def load_from_file(self, file, filter_status=[], filter_platform=[]):
Anas Nashifce2b4182020-03-24 14:40:28 -04003149 try:
3150 with open(file, "r") as fp:
3151 cr = csv.DictReader(fp)
3152 instance_list = []
3153 for row in cr:
3154 if row["status"] in filter_status:
3155 continue
3156 test = row["test"]
3157
3158 platform = self.get_platform(row["platform"])
Anas Nashif82a6a462020-11-02 10:47:36 -05003159 if filter_platform and platform.name not in filter_platform:
3160 continue
Anas Nashifce2b4182020-03-24 14:40:28 -04003161 instance = TestInstance(self.testcases[test], platform, self.outdir)
Anas Nashif405f1b62020-07-27 12:27:13 -04003162 if self.device_testing:
3163 tfilter = 'runnable'
3164 else:
3165 tfilter = 'buildable'
3166 instance.run = instance.check_runnable(
Anas Nashifce2b4182020-03-24 14:40:28 -04003167 self.enable_slow,
Anas Nashif405f1b62020-07-27 12:27:13 -04003168 tfilter,
Anas Nashifce8c12e2020-05-21 09:11:40 -04003169 self.fixtures
Anas Nashifce2b4182020-03-24 14:40:28 -04003170 )
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02003171 instance.create_overlay(platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform)
Anas Nashifce2b4182020-03-24 14:40:28 -04003172 instance_list.append(instance)
3173 self.add_instances(instance_list)
3174
3175 except KeyError as e:
3176 logger.error("Key error while parsing tests file.({})".format(str(e)))
3177 sys.exit(2)
3178
3179 except FileNotFoundError as e:
3180 logger.error("Couldn't find input file with list of tests. ({})".format(e))
3181 sys.exit(2)
3182
3183 def apply_filters(self, **kwargs):
3184
3185 toolchain = self.get_toolchain()
3186
3187 discards = {}
3188 platform_filter = kwargs.get('platform')
Anas Nashifaff616d2020-04-17 21:24:57 -04003189 exclude_platform = kwargs.get('exclude_platform', [])
3190 testcase_filter = kwargs.get('run_individual_tests', [])
Anas Nashifce2b4182020-03-24 14:40:28 -04003191 arch_filter = kwargs.get('arch')
3192 tag_filter = kwargs.get('tag')
3193 exclude_tag = kwargs.get('exclude_tag')
3194 all_filter = kwargs.get('all')
Anas Nashif405f1b62020-07-27 12:27:13 -04003195 runnable = kwargs.get('runnable')
Anas Nashifce2b4182020-03-24 14:40:28 -04003196 force_toolchain = kwargs.get('force_toolchain')
Anas Nashif1a5defa2020-05-01 14:57:00 -04003197 force_platform = kwargs.get('force_platform')
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04003198 emu_filter = kwargs.get('emulation_only')
Anas Nashifce2b4182020-03-24 14:40:28 -04003199
3200 logger.debug("platform filter: " + str(platform_filter))
3201 logger.debug(" arch_filter: " + str(arch_filter))
3202 logger.debug(" tag_filter: " + str(tag_filter))
3203 logger.debug(" exclude_tag: " + str(exclude_tag))
3204
3205 default_platforms = False
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04003206 emulation_platforms = False
Anas Nashifce2b4182020-03-24 14:40:28 -04003207
Anas Nashifce2b4182020-03-24 14:40:28 -04003208
3209 if all_filter:
3210 logger.info("Selecting all possible platforms per test case")
3211 # When --all used, any --platform arguments ignored
3212 platform_filter = []
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04003213 elif not platform_filter and not emu_filter:
Anas Nashifce2b4182020-03-24 14:40:28 -04003214 logger.info("Selecting default platforms per test case")
3215 default_platforms = True
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04003216 elif emu_filter:
3217 logger.info("Selecting emulation platforms per test case")
3218 emulation_platforms = True
Anas Nashifce2b4182020-03-24 14:40:28 -04003219
Anas Nashif3b939da2020-11-24 13:21:27 -05003220 if platform_filter:
3221 platforms = list(filter(lambda p: p.name in platform_filter, self.platforms))
3222 elif emu_filter:
3223 platforms = list(filter(lambda p: p.simulation != 'na', self.platforms))
3224 elif arch_filter:
3225 platforms = list(filter(lambda p: p.arch in arch_filter, self.platforms))
3226 elif default_platforms:
3227 platforms = list(filter(lambda p: p.default, self.platforms))
3228 else:
3229 platforms = self.platforms
3230
Anas Nashifce2b4182020-03-24 14:40:28 -04003231 logger.info("Building initial testcase list...")
3232
3233 for tc_name, tc in self.testcases.items():
Anas Nashif827ecb72021-01-14 09:34:29 -05003234
3235 if tc.build_on_all and not platform_filter:
3236 platform_scope = self.platforms
Anas Nashife618a592021-03-03 14:05:54 -05003237 elif tc.integration_platforms and self.integration:
3238 platform_scope = list(filter(lambda item: item.name in tc.integration_platforms, \
3239 self.platforms))
Anas Nashif827ecb72021-01-14 09:34:29 -05003240 else:
3241 platform_scope = platforms
3242
Kumar Galad590e0d2021-04-02 10:18:01 -05003243 integration = self.integration and tc.integration_platforms
3244
3245 # If there isn't any overlap between the platform_allow list and the platform_scope
3246 # we set the scope to the platform_allow list
3247 if tc.platform_allow and not platform_filter and not integration:
3248 a = set(platform_scope)
3249 b = set(filter(lambda item: item.name in tc.platform_allow, self.platforms))
3250 c = a.intersection(b)
3251 if not c:
3252 platform_scope = list(filter(lambda item: item.name in tc.platform_allow, \
3253 self.platforms))
3254
Anas Nashifce2b4182020-03-24 14:40:28 -04003255 # list of instances per testcase, aka configurations.
3256 instance_list = []
Anas Nashif827ecb72021-01-14 09:34:29 -05003257 for plat in platform_scope:
Anas Nashifce2b4182020-03-24 14:40:28 -04003258 instance = TestInstance(tc, plat, self.outdir)
Anas Nashif405f1b62020-07-27 12:27:13 -04003259 if runnable:
3260 tfilter = 'runnable'
3261 else:
3262 tfilter = 'buildable'
3263
3264 instance.run = instance.check_runnable(
Anas Nashifce2b4182020-03-24 14:40:28 -04003265 self.enable_slow,
Anas Nashif405f1b62020-07-27 12:27:13 -04003266 tfilter,
Anas Nashifce8c12e2020-05-21 09:11:40 -04003267 self.fixtures
Anas Nashifce2b4182020-03-24 14:40:28 -04003268 )
Anas Nashiff86bc052020-11-19 16:05:03 -05003269
Anas Nashiff04461e2020-06-29 10:07:02 -04003270 for t in tc.cases:
3271 instance.results[t] = None
Anas Nashif3b86f132020-05-21 10:35:33 -04003272
Anas Nashif8305d1b2020-11-26 11:55:02 -05003273 if runnable and self.duts:
3274 for h in self.duts:
Anas Nashif531fe892020-09-11 13:56:33 -04003275 if h.platform == plat.name:
3276 if tc.harness_config.get('fixture') in h.fixtures:
Anas Nashif3b86f132020-05-21 10:35:33 -04003277 instance.run = True
3278
Anas Nashif1a5defa2020-05-01 14:57:00 -04003279 if not force_platform and plat.name in exclude_platform:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003280 discards[instance] = discards.get(instance, "Platform is excluded on command line.")
Anas Nashifce2b4182020-03-24 14:40:28 -04003281
3282 if (plat.arch == "unit") != (tc.type == "unit"):
3283 # Discard silently
3284 continue
3285
Anas Nashif405f1b62020-07-27 12:27:13 -04003286 if runnable and not instance.run:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003287 discards[instance] = discards.get(instance, "Not runnable on device")
Anas Nashifce2b4182020-03-24 14:40:28 -04003288
Anas Nashif1636c312020-05-28 08:02:54 -04003289 if self.integration and tc.integration_platforms and plat.name not in tc.integration_platforms:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003290 discards[instance] = discards.get(instance, "Not part of integration platforms")
Anas Nashif1636c312020-05-28 08:02:54 -04003291
Anas Nashifce2b4182020-03-24 14:40:28 -04003292 if tc.skip:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003293 discards[instance] = discards.get(instance, "Skip filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04003294
Anas Nashifce2b4182020-03-24 14:40:28 -04003295 if tag_filter and not tc.tags.intersection(tag_filter):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003296 discards[instance] = discards.get(instance, "Command line testcase tag filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04003297
3298 if exclude_tag and tc.tags.intersection(exclude_tag):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003299 discards[instance] = discards.get(instance, "Command line testcase exclude filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04003300
3301 if testcase_filter and tc_name not in testcase_filter:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003302 discards[instance] = discards.get(instance, "Testcase name filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04003303
3304 if arch_filter and plat.arch not in arch_filter:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003305 discards[instance] = discards.get(instance, "Command line testcase arch filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04003306
Anas Nashif1a5defa2020-05-01 14:57:00 -04003307 if not force_platform:
Anas Nashifce2b4182020-03-24 14:40:28 -04003308
Anas Nashifdca317c2020-08-26 11:28:25 -04003309 if tc.arch_allow and plat.arch not in tc.arch_allow:
3310 discards[instance] = discards.get(instance, "Not in test case arch allow list")
Anas Nashifce2b4182020-03-24 14:40:28 -04003311
Anas Nashif1a5defa2020-05-01 14:57:00 -04003312 if tc.arch_exclude and plat.arch in tc.arch_exclude:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003313 discards[instance] = discards.get(instance, "In test case arch exclude")
Anas Nashif1a5defa2020-05-01 14:57:00 -04003314
3315 if tc.platform_exclude and plat.name in tc.platform_exclude:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003316 discards[instance] = discards.get(instance, "In test case platform exclude")
Anas Nashifce2b4182020-03-24 14:40:28 -04003317
3318 if tc.toolchain_exclude and toolchain in tc.toolchain_exclude:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003319 discards[instance] = discards.get(instance, "In test case toolchain exclude")
Anas Nashifce2b4182020-03-24 14:40:28 -04003320
3321 if platform_filter and plat.name not in platform_filter:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003322 discards[instance] = discards.get(instance, "Command line platform filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04003323
Anas Nashifdca317c2020-08-26 11:28:25 -04003324 if tc.platform_allow and plat.name not in tc.platform_allow:
3325 discards[instance] = discards.get(instance, "Not in testcase platform allow list")
Anas Nashifce2b4182020-03-24 14:40:28 -04003326
Anas Nashifdca317c2020-08-26 11:28:25 -04003327 if tc.toolchain_allow and toolchain not in tc.toolchain_allow:
3328 discards[instance] = discards.get(instance, "Not in testcase toolchain allow list")
Anas Nashifce2b4182020-03-24 14:40:28 -04003329
3330 if not plat.env_satisfied:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003331 discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env)))
Anas Nashifce2b4182020-03-24 14:40:28 -04003332
3333 if not force_toolchain \
3334 and toolchain and (toolchain not in plat.supported_toolchains) \
Maciej Perkowskiedb23d72021-09-09 16:16:49 +02003335 and "host" not in plat.supported_toolchains \
Anas Nashifce2b4182020-03-24 14:40:28 -04003336 and tc.type != 'unit':
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003337 discards[instance] = discards.get(instance, "Not supported by the toolchain")
Anas Nashifce2b4182020-03-24 14:40:28 -04003338
3339 if plat.ram < tc.min_ram:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003340 discards[instance] = discards.get(instance, "Not enough RAM")
Anas Nashifce2b4182020-03-24 14:40:28 -04003341
3342 if tc.depends_on:
3343 dep_intersection = tc.depends_on.intersection(set(plat.supported))
3344 if dep_intersection != set(tc.depends_on):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003345 discards[instance] = discards.get(instance, "No hardware support")
Anas Nashifce2b4182020-03-24 14:40:28 -04003346
3347 if plat.flash < tc.min_flash:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003348 discards[instance] = discards.get(instance, "Not enough FLASH")
Anas Nashifce2b4182020-03-24 14:40:28 -04003349
3350 if set(plat.ignore_tags) & tc.tags:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003351 discards[instance] = discards.get(instance, "Excluded tags per platform (exclude_tags)")
Anas Nashife8e367a2020-07-16 16:27:04 -04003352
Anas Nashif555fc6d2020-07-30 07:23:54 -04003353 if plat.only_tags and not set(plat.only_tags) & tc.tags:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003354 discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
Anas Nashifce2b4182020-03-24 14:40:28 -04003355
Maciej Perkowski9c6dfce2021-03-11 13:18:33 +01003356 test_configuration = ".".join([instance.platform.name,
3357 instance.testcase.id])
3358 # skip quarantined tests
3359 if test_configuration in self.quarantine and not self.quarantine_verify:
3360 discards[instance] = discards.get(instance,
3361 f"Quarantine: {self.quarantine[test_configuration]}")
3362 # run only quarantined test to verify their statuses (skip everything else)
3363 if self.quarantine_verify and test_configuration not in self.quarantine:
3364 discards[instance] = discards.get(instance, "Not under quarantine")
3365
Anas Nashifce2b4182020-03-24 14:40:28 -04003366 # if nothing stopped us until now, it means this configuration
3367 # needs to be added.
3368 instance_list.append(instance)
3369
3370 # no configurations, so jump to next testcase
3371 if not instance_list:
3372 continue
3373
Anas Nashifb18f3112020-12-07 11:40:19 -05003374 # if twister was launched with no platform options at all, we
Anas Nashifce2b4182020-03-24 14:40:28 -04003375 # take all default platforms
Anas Nashife618a592021-03-03 14:05:54 -05003376 if default_platforms and not tc.build_on_all and not integration:
Anas Nashifdca317c2020-08-26 11:28:25 -04003377 if tc.platform_allow:
Anas Nashifce2b4182020-03-24 14:40:28 -04003378 a = set(self.default_platforms)
Anas Nashifdca317c2020-08-26 11:28:25 -04003379 b = set(tc.platform_allow)
Anas Nashifce2b4182020-03-24 14:40:28 -04003380 c = a.intersection(b)
3381 if c:
3382 aa = list(filter(lambda tc: tc.platform.name in c, instance_list))
3383 self.add_instances(aa)
3384 else:
Kumar Galad590e0d2021-04-02 10:18:01 -05003385 self.add_instances(instance_list)
Anas Nashifce2b4182020-03-24 14:40:28 -04003386 else:
3387 instances = list(filter(lambda tc: tc.platform.default, instance_list))
3388 self.add_instances(instances)
Anas Nashif4bccc1d2021-03-12 18:21:29 -05003389 elif integration:
Anas Nashife618a592021-03-03 14:05:54 -05003390 instances = list(filter(lambda item: item.platform.name in tc.integration_platforms, instance_list))
3391 self.add_instances(instances)
3392
3393
Anas Nashifce2b4182020-03-24 14:40:28 -04003394
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04003395 elif emulation_platforms:
3396 self.add_instances(instance_list)
3397 for instance in list(filter(lambda inst: not inst.platform.simulation != 'na', instance_list)):
3398 discards[instance] = discards.get(instance, "Not an emulated platform")
Anas Nashifce2b4182020-03-24 14:40:28 -04003399 else:
3400 self.add_instances(instance_list)
3401
3402 for _, case in self.instances.items():
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02003403 case.create_overlay(case.platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform)
Anas Nashifce2b4182020-03-24 14:40:28 -04003404
3405 self.discards = discards
3406 self.selected_platforms = set(p.platform.name for p in self.instances.values())
3407
Maciej Perkowski2253c6e2021-10-21 14:05:54 +02003408 remove_from_discards = [] # configurations to be removed from discards.
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003409 for instance in self.discards:
3410 instance.reason = self.discards[instance]
Maciej Perkowskid08ee922021-03-30 14:12:02 +02003411 # If integration mode is on all skips on integration_platforms are treated as errors.
Maciej Perkowskid8716032021-07-22 15:41:18 +02003412 if self.integration and instance.platform.name in instance.testcase.integration_platforms \
3413 and "Quarantine" not in instance.reason:
Maciej Perkowskid08ee922021-03-30 14:12:02 +02003414 instance.status = "error"
3415 instance.reason += " but is one of the integration platforms"
3416 instance.fill_results_by_status()
3417 self.instances[instance.name] = instance
Maciej Perkowski2253c6e2021-10-21 14:05:54 +02003418 # Such configuration has to be removed from discards to make sure it won't get skipped
3419 remove_from_discards.append(instance)
Maciej Perkowskid08ee922021-03-30 14:12:02 +02003420 else:
3421 instance.status = "skipped"
3422 instance.fill_results_by_status()
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003423
Maciej Perkowskib6a69aa2021-02-04 16:05:21 +01003424 self.filtered_platforms = set(p.platform.name for p in self.instances.values()
3425 if p.status != "skipped" )
3426
Maciej Perkowski2253c6e2021-10-21 14:05:54 +02003427 # Remove from discards configururations that must not be discarded (e.g. integration_platforms when --integration was used)
3428 for instance in remove_from_discards:
3429 del self.discards[instance]
3430
Anas Nashifce2b4182020-03-24 14:40:28 -04003431 return discards
3432
3433 def add_instances(self, instance_list):
3434 for instance in instance_list:
3435 self.instances[instance.name] = instance
3436
Anas Nashif531fe892020-09-11 13:56:33 -04003437 @staticmethod
3438 def calc_one_elf_size(instance):
3439 if instance.status not in ["error", "failed", "skipped"]:
3440 if instance.platform.type != "native":
3441 size_calc = instance.calculate_sizes()
3442 instance.metrics["ram_size"] = size_calc.get_ram_size()
3443 instance.metrics["rom_size"] = size_calc.get_rom_size()
3444 instance.metrics["unrecognized"] = size_calc.unrecognized_sections()
3445 else:
3446 instance.metrics["ram_size"] = 0
3447 instance.metrics["rom_size"] = 0
3448 instance.metrics["unrecognized"] = []
3449
3450 instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0
3451
3452 def add_tasks_to_queue(self, pipeline, build_only=False, test_only=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04003453 for instance in self.instances.values():
Anas Nashif405f1b62020-07-27 12:27:13 -04003454 if build_only:
3455 instance.run = False
3456
Andreas Vibeto32fbf8e2021-07-16 10:01:49 +02003457 if instance.status not in ['passed', 'skipped', 'error']:
3458 logger.debug(f"adding {instance.name}")
3459 instance.status = None
3460 if test_only and instance.run:
3461 pipeline.put({"op": "run", "test": instance})
3462 else:
Anas Nashifce2b4182020-03-24 14:40:28 -04003463 pipeline.put({"op": "cmake", "test": instance})
Andreas Vibeto32fbf8e2021-07-16 10:01:49 +02003464 # If the instance got 'error' status before, proceed to the report stage
3465 if instance.status == "error":
3466 pipeline.put({"op": "report", "test": instance})
Anas Nashifce2b4182020-03-24 14:40:28 -04003467
Anas Nashif531fe892020-09-11 13:56:33 -04003468 def pipeline_mgr(self, pipeline, done_queue, lock, results):
3469 while True:
3470 try:
3471 task = pipeline.get_nowait()
3472 except queue.Empty:
3473 break
3474 else:
3475 test = task['test']
3476 pb = ProjectBuilder(self,
3477 test,
3478 lsan=self.enable_lsan,
3479 asan=self.enable_asan,
3480 ubsan=self.enable_ubsan,
3481 coverage=self.enable_coverage,
3482 extra_args=self.extra_args,
3483 device_testing=self.device_testing,
3484 cmake_only=self.cmake_only,
3485 cleanup=self.cleanup,
3486 valgrind=self.enable_valgrind,
3487 inline_logs=self.inline_logs,
3488 generator=self.generator,
3489 generator_cmd=self.generator_cmd,
3490 verbose=self.verbose,
Anas Nashiff68146f2020-11-28 11:07:06 -05003491 warnings_as_errors=self.warnings_as_errors,
3492 overflow_as_errors=self.overflow_as_errors
Anas Nashif531fe892020-09-11 13:56:33 -04003493 )
3494 pb.process(pipeline, done_queue, task, lock, results)
Anas Nashifce2b4182020-03-24 14:40:28 -04003495
Anas Nashif531fe892020-09-11 13:56:33 -04003496 return True
Anas Nashifdc43c292020-07-09 09:46:45 -04003497
Anas Nashif531fe892020-09-11 13:56:33 -04003498 def execute(self, pipeline, done, results):
3499 lock = Lock()
Anas Nashifce2b4182020-03-24 14:40:28 -04003500 logger.info("Adding tasks to the queue...")
Anas Nashif531fe892020-09-11 13:56:33 -04003501 self.add_tasks_to_queue(pipeline, self.build_only, self.test_only)
3502 logger.info("Added initial list of jobs to queue")
Anas Nashifce2b4182020-03-24 14:40:28 -04003503
Anas Nashif531fe892020-09-11 13:56:33 -04003504 processes = []
3505 for job in range(self.jobs):
3506 logger.debug(f"Launch process {job}")
3507 p = Process(target=self.pipeline_mgr, args=(pipeline, done, lock, results, ))
3508 processes.append(p)
3509 p.start()
Anas Nashifce2b4182020-03-24 14:40:28 -04003510
Anas Nashif2d489172020-12-09 09:06:57 -05003511 try:
3512 for p in processes:
3513 p.join()
3514 except KeyboardInterrupt:
3515 logger.info("Execution interrupted")
3516 for p in processes:
3517 p.terminate()
Anas Nashifce2b4182020-03-24 14:40:28 -04003518
Anas Nashif531fe892020-09-11 13:56:33 -04003519 # FIXME: This needs to move out.
Anas Nashifce2b4182020-03-24 14:40:28 -04003520 if self.enable_size_report and not self.cmake_only:
3521 # Parallelize size calculation
3522 executor = concurrent.futures.ThreadPoolExecutor(self.jobs)
Anas Nashif531fe892020-09-11 13:56:33 -04003523 futures = [executor.submit(self.calc_one_elf_size, instance)
Anas Nashifce2b4182020-03-24 14:40:28 -04003524 for instance in self.instances.values()]
3525 concurrent.futures.wait(futures)
3526 else:
3527 for instance in self.instances.values():
3528 instance.metrics["ram_size"] = 0
3529 instance.metrics["rom_size"] = 0
3530 instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0
3531 instance.metrics["unrecognized"] = []
3532
Anas Nashif531fe892020-09-11 13:56:33 -04003533 return results
3534
Anas Nashifce2b4182020-03-24 14:40:28 -04003535 def discard_report(self, filename):
3536
3537 try:
Aastha Groverdcbd9152020-06-16 10:19:51 -07003538 if not self.discards:
Anas Nashif45943702020-12-11 17:55:15 -05003539 raise TwisterRuntimeError("apply_filters() hasn't been run!")
Anas Nashifce2b4182020-03-24 14:40:28 -04003540 except Exception as e:
3541 logger.error(str(e))
3542 sys.exit(2)
Anas Nashifce2b4182020-03-24 14:40:28 -04003543 with open(filename, "wt") as csvfile:
3544 fieldnames = ["test", "arch", "platform", "reason"]
3545 cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
3546 cw.writeheader()
3547 for instance, reason in sorted(self.discards.items()):
3548 rowdict = {"test": instance.testcase.name,
3549 "arch": instance.platform.arch,
3550 "platform": instance.platform.name,
3551 "reason": reason}
3552 cw.writerow(rowdict)
3553
Anas Nashif6915adf2020-04-22 09:39:42 -04003554 def target_report(self, outdir, suffix, append=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04003555 platforms = {inst.platform.name for _, inst in self.instances.items()}
3556 for platform in platforms:
Anas Nashif6915adf2020-04-22 09:39:42 -04003557 if suffix:
3558 filename = os.path.join(outdir,"{}_{}.xml".format(platform, suffix))
3559 else:
3560 filename = os.path.join(outdir,"{}.xml".format(platform))
Maciej Perkowski060e00d2020-10-13 18:59:37 +02003561 self.xunit_report(filename, platform, full_report=True,
3562 append=append, version=self.version)
Anas Nashifce2b4182020-03-24 14:40:28 -04003563
Anas Nashif90415502020-04-11 22:15:04 -04003564
3565 @staticmethod
3566 def process_log(log_file):
3567 filtered_string = ""
3568 if os.path.exists(log_file):
3569 with open(log_file, "rb") as f:
3570 log = f.read().decode("utf-8")
3571 filtered_string = ''.join(filter(lambda x: x in string.printable, log))
3572
3573 return filtered_string
3574
Anas Nashifa53c8132020-05-05 09:32:46 -04003575
Maciej Perkowski725d19b2020-10-13 14:15:12 +02003576 def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA"):
Anas Nashifa53c8132020-05-05 09:32:46 -04003577 total = 0
Anas Nashifb517b1f2020-12-11 07:38:49 -05003578 fails = passes = errors = skips = 0
Anas Nashifa53c8132020-05-05 09:32:46 -04003579 if platform:
3580 selected = [platform]
Anas Nashifb517b1f2020-12-11 07:38:49 -05003581 logger.info(f"Writing target report for {platform}...")
Anas Nashifa53c8132020-05-05 09:32:46 -04003582 else:
Anas Nashifb517b1f2020-12-11 07:38:49 -05003583 logger.info(f"Writing xunit report {filename}...")
Anas Nashifa53c8132020-05-05 09:32:46 -04003584 selected = self.selected_platforms
Anas Nashif90415502020-04-11 22:15:04 -04003585
Anas Nashif90415502020-04-11 22:15:04 -04003586 if os.path.exists(filename) and append:
3587 tree = ET.parse(filename)
3588 eleTestsuites = tree.getroot()
Anas Nashif90415502020-04-11 22:15:04 -04003589 else:
Anas Nashifce2b4182020-03-24 14:40:28 -04003590 eleTestsuites = ET.Element('testsuites')
Anas Nashifce2b4182020-03-24 14:40:28 -04003591
Anas Nashifa53c8132020-05-05 09:32:46 -04003592 for p in selected:
3593 inst = self.get_platform_instances(p)
3594 fails = 0
3595 passes = 0
3596 errors = 0
3597 skips = 0
3598 duration = 0
3599
3600 for _, instance in inst.items():
3601 handler_time = instance.metrics.get('handler_time', 0)
3602 duration += handler_time
Anas Nashif405f1b62020-07-27 12:27:13 -04003603 if full_report and instance.run:
Anas Nashifa53c8132020-05-05 09:32:46 -04003604 for k in instance.results.keys():
3605 if instance.results[k] == 'PASS':
3606 passes += 1
3607 elif instance.results[k] == 'BLOCK':
3608 errors += 1
Anas Nashifbe1025a2020-10-31 08:04:48 -04003609 elif instance.results[k] == 'SKIP' or instance.status in ['skipped']:
Anas Nashifa53c8132020-05-05 09:32:46 -04003610 skips += 1
3611 else:
3612 fails += 1
3613 else:
Maciej Perkowskif050a992021-02-24 13:43:05 +01003614 if instance.status in ["error", "failed", "timeout", "flash_error"]:
Anas Nashifa53c8132020-05-05 09:32:46 -04003615 if instance.reason in ['build_error', 'handler_crash']:
3616 errors += 1
3617 else:
3618 fails += 1
3619 elif instance.status == 'skipped':
3620 skips += 1
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003621 elif instance.status == 'passed':
Anas Nashifa53c8132020-05-05 09:32:46 -04003622 passes += 1
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003623 else:
Anas Nashif696476e2020-11-24 08:42:46 -05003624 if instance.status:
3625 logger.error(f"{instance.name}: Unknown status {instance.status}")
3626 else:
3627 logger.error(f"{instance.name}: No status")
Anas Nashifa53c8132020-05-05 09:32:46 -04003628
3629 total = (errors + passes + fails + skips)
3630 # do not produce a report if no tests were actually run (only built)
3631 if total == 0:
Anas Nashif90415502020-04-11 22:15:04 -04003632 continue
Anas Nashifce2b4182020-03-24 14:40:28 -04003633
Anas Nashifa53c8132020-05-05 09:32:46 -04003634 run = p
3635 eleTestsuite = None
3636
3637 # When we re-run the tests, we re-use the results and update only with
3638 # the newly run tests.
3639 if os.path.exists(filename) and append:
Anas Nashiff04461e2020-06-29 10:07:02 -04003640 ts = eleTestsuites.findall(f'testsuite/[@name="{p}"]')
3641 if ts:
3642 eleTestsuite = ts[0]
3643 eleTestsuite.attrib['failures'] = "%d" % fails
3644 eleTestsuite.attrib['errors'] = "%d" % errors
Christian Taedckeb2be8042020-08-12 14:21:13 +02003645 eleTestsuite.attrib['skipped'] = "%d" % skips
Anas Nashiff04461e2020-06-29 10:07:02 -04003646 else:
3647 logger.info(f"Did not find any existing results for {p}")
3648 eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite',
3649 name=run, time="%f" % duration,
3650 tests="%d" % (total),
3651 failures="%d" % fails,
Christian Taedckeb2be8042020-08-12 14:21:13 +02003652 errors="%d" % (errors), skipped="%s" % (skips))
Maciej Perkowski725d19b2020-10-13 14:15:12 +02003653 eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
3654 # Multiple 'property' can be added to 'properties'
3655 # differing by name and value
Maciej Perkowski060e00d2020-10-13 18:59:37 +02003656 ET.SubElement(eleTSPropetries, 'property', name="version", value=version)
Anas Nashiff04461e2020-06-29 10:07:02 -04003657
Anas Nashif90415502020-04-11 22:15:04 -04003658 else:
Anas Nashifa53c8132020-05-05 09:32:46 -04003659 eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite',
3660 name=run, time="%f" % duration,
3661 tests="%d" % (total),
3662 failures="%d" % fails,
Christian Taedckeb2be8042020-08-12 14:21:13 +02003663 errors="%d" % (errors), skipped="%s" % (skips))
Maciej Perkowski725d19b2020-10-13 14:15:12 +02003664 eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
3665 # Multiple 'property' can be added to 'properties'
3666 # differing by name and value
Maciej Perkowski060e00d2020-10-13 18:59:37 +02003667 ET.SubElement(eleTSPropetries, 'property', name="version", value=version)
Anas Nashif90415502020-04-11 22:15:04 -04003668
Anas Nashifa53c8132020-05-05 09:32:46 -04003669 for _, instance in inst.items():
3670 if full_report:
3671 tname = os.path.basename(instance.testcase.name)
3672 else:
3673 tname = instance.testcase.id
Anas Nashifa53c8132020-05-05 09:32:46 -04003674 handler_time = instance.metrics.get('handler_time', 0)
3675
3676 if full_report:
3677 for k in instance.results.keys():
Anas Nashifa53c8132020-05-05 09:32:46 -04003678 # remove testcases that are being re-run from exiting reports
3679 for tc in eleTestsuite.findall(f'testcase/[@name="{k}"]'):
3680 eleTestsuite.remove(tc)
3681
3682 classname = ".".join(tname.split(".")[:2])
3683 eleTestcase = ET.SubElement(
3684 eleTestsuite, 'testcase',
3685 classname=classname,
3686 name="%s" % (k), time="%f" % handler_time)
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003687 if instance.results[k] in ['FAIL', 'BLOCK'] or \
Anas Nashif405f1b62020-07-27 12:27:13 -04003688 (not instance.run and instance.status in ["error", "failed", "timeout"]):
Anas Nashifa53c8132020-05-05 09:32:46 -04003689 if instance.results[k] == 'FAIL':
3690 el = ET.SubElement(
3691 eleTestcase,
3692 'failure',
3693 type="failure",
3694 message="failed")
3695 else:
3696 el = ET.SubElement(
3697 eleTestcase,
3698 'error',
3699 type="failure",
Maciej Perkowskif050a992021-02-24 13:43:05 +01003700 message=instance.reason)
Jingru Wang99b07202021-01-29 12:42:32 +08003701 log_root = os.path.join(self.outdir, instance.platform.name, instance.testcase.name)
3702 log_file = os.path.join(log_root, "handler.log")
Anas Nashifa53c8132020-05-05 09:32:46 -04003703 el.text = self.process_log(log_file)
3704
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003705 elif instance.results[k] == 'PASS' \
Anas Nashif405f1b62020-07-27 12:27:13 -04003706 or (not instance.run and instance.status in ["passed"]):
Anas Nashiff04461e2020-06-29 10:07:02 -04003707 pass
Anas Nashifbe1025a2020-10-31 08:04:48 -04003708 elif instance.results[k] == 'SKIP' or (instance.status in ["skipped"]):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003709 el = ET.SubElement(eleTestcase, 'skipped', type="skipped", message=instance.reason)
Anas Nashiff04461e2020-06-29 10:07:02 -04003710 else:
Anas Nashifce2b4182020-03-24 14:40:28 -04003711 el = ET.SubElement(
3712 eleTestcase,
Anas Nashiff04461e2020-06-29 10:07:02 -04003713 'error',
3714 type="error",
3715 message=f"{instance.reason}")
Anas Nashifa53c8132020-05-05 09:32:46 -04003716 else:
3717 if platform:
3718 classname = ".".join(instance.testcase.name.split(".")[:2])
3719 else:
3720 classname = p + ":" + ".".join(instance.testcase.name.split(".")[:2])
Anas Nashifce2b4182020-03-24 14:40:28 -04003721
Anas Nashiff04461e2020-06-29 10:07:02 -04003722 # remove testcases that are being re-run from exiting reports
Sebastian Wezel148dbdc2021-02-22 13:04:56 +01003723 for tc in eleTestsuite.findall(f'testcase/[@classname="{classname}"][@name="{instance.testcase.name}"]'):
Anas Nashiff04461e2020-06-29 10:07:02 -04003724 eleTestsuite.remove(tc)
3725
Anas Nashifa53c8132020-05-05 09:32:46 -04003726 eleTestcase = ET.SubElement(eleTestsuite, 'testcase',
3727 classname=classname,
3728 name="%s" % (instance.testcase.name),
3729 time="%f" % handler_time)
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003730
Maciej Perkowskif050a992021-02-24 13:43:05 +01003731 if instance.status in ["error", "failed", "timeout", "flash_error"]:
Anas Nashifa53c8132020-05-05 09:32:46 -04003732 failure = ET.SubElement(
Anas Nashifce2b4182020-03-24 14:40:28 -04003733 eleTestcase,
Anas Nashifa53c8132020-05-05 09:32:46 -04003734 'failure',
3735 type="failure",
Maciej Perkowskib2fa99c2020-05-21 14:45:29 +02003736 message=instance.reason)
Anas Nashiff04461e2020-06-29 10:07:02 -04003737
Jingru Wang99b07202021-01-29 12:42:32 +08003738 log_root = ("%s/%s/%s" % (self.outdir, instance.platform.name, instance.testcase.name))
3739 bl = os.path.join(log_root, "build.log")
3740 hl = os.path.join(log_root, "handler.log")
Anas Nashifa53c8132020-05-05 09:32:46 -04003741 log_file = bl
3742 if instance.reason != 'Build error':
3743 if os.path.exists(hl):
3744 log_file = hl
3745 else:
3746 log_file = bl
Anas Nashifce2b4182020-03-24 14:40:28 -04003747
Anas Nashifa53c8132020-05-05 09:32:46 -04003748 failure.text = self.process_log(log_file)
Anas Nashifce2b4182020-03-24 14:40:28 -04003749
Anas Nashifa53c8132020-05-05 09:32:46 -04003750 elif instance.status == "skipped":
3751 ET.SubElement(eleTestcase, 'skipped', type="skipped", message="Skipped")
Anas Nashifce2b4182020-03-24 14:40:28 -04003752
3753 result = ET.tostring(eleTestsuites)
3754 with open(filename, 'wb') as report:
3755 report.write(result)
3756
Anas Nashif1c2f1272020-07-23 09:56:01 -04003757 return fails, passes, errors, skips
Anas Nashif90415502020-04-11 22:15:04 -04003758
Anas Nashifce2b4182020-03-24 14:40:28 -04003759 def csv_report(self, filename):
3760 with open(filename, "wt") as csvfile:
3761 fieldnames = ["test", "arch", "platform", "status",
3762 "extra_args", "handler", "handler_time", "ram_size",
3763 "rom_size"]
3764 cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
3765 cw.writeheader()
3766 for instance in self.instances.values():
3767 rowdict = {"test": instance.testcase.name,
3768 "arch": instance.platform.arch,
3769 "platform": instance.platform.name,
3770 "extra_args": " ".join(instance.testcase.extra_args),
3771 "handler": instance.platform.simulation}
3772
3773 rowdict["status"] = instance.status
Anas Nashiff04461e2020-06-29 10:07:02 -04003774 if instance.status not in ["error", "failed", "timeout"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04003775 if instance.handler:
3776 rowdict["handler_time"] = instance.metrics.get("handler_time", 0)
3777 ram_size = instance.metrics.get("ram_size", 0)
3778 rom_size = instance.metrics.get("rom_size", 0)
3779 rowdict["ram_size"] = ram_size
3780 rowdict["rom_size"] = rom_size
3781 cw.writerow(rowdict)
3782
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003783 def json_report(self, filename, append=False, version="NA"):
Anas Nashifb517b1f2020-12-11 07:38:49 -05003784 logger.info(f"Writing JSON report {filename}")
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003785 report = {}
3786 selected = self.selected_platforms
3787 report["environment"] = {"os": os.name,
3788 "zephyr_version": version,
3789 "toolchain": self.get_toolchain()
3790 }
3791 json_data = {}
3792 if os.path.exists(filename) and append:
3793 with open(filename, 'r') as json_file:
3794 json_data = json.load(json_file)
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003795
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003796 suites = json_data.get("testsuites", [])
3797 if suites:
3798 suite = suites[0]
3799 testcases = suite.get("testcases", [])
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003800 else:
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003801 suite = {}
3802 testcases = []
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003803
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003804 for p in selected:
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003805 inst = self.get_platform_instances(p)
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003806 for _, instance in inst.items():
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003807 testcase = {}
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003808 handler_log = os.path.join(instance.build_dir, "handler.log")
3809 build_log = os.path.join(instance.build_dir, "build.log")
3810 device_log = os.path.join(instance.build_dir, "device.log")
3811
3812 handler_time = instance.metrics.get('handler_time', 0)
3813 ram_size = instance.metrics.get ("ram_size", 0)
3814 rom_size = instance.metrics.get("rom_size",0)
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003815 for k in instance.results.keys():
3816 testcases = list(filter(lambda d: not (d.get('testcase') == k and d.get('platform') == p), testcases ))
3817 testcase = {"testcase": k,
3818 "arch": instance.platform.arch,
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003819 "platform": p,
3820 }
Maciej Perkowski484ef672021-04-15 11:37:39 +02003821 if ram_size:
3822 testcase["ram_size"] = ram_size
3823 if rom_size:
3824 testcase["rom_size"] = rom_size
3825
Anas Nashif55c3dde2021-09-16 08:54:19 -04003826 if instance.results[k] in ["PASS"] or instance.status == 'passed':
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003827 testcase["status"] = "passed"
3828 if instance.handler:
3829 testcase["execution_time"] = handler_time
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003830
Anas Nashif55c3dde2021-09-16 08:54:19 -04003831 elif instance.results[k] in ['FAIL', 'BLOCK'] or instance.status in ["error", "failed", "timeout", "flash_error"]:
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003832 testcase["status"] = "failed"
3833 testcase["reason"] = instance.reason
3834 testcase["execution_time"] = handler_time
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003835 if os.path.exists(handler_log):
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003836 testcase["test_output"] = self.process_log(handler_log)
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003837 elif os.path.exists(device_log):
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003838 testcase["device_log"] = self.process_log(device_log)
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003839 else:
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003840 testcase["build_log"] = self.process_log(build_log)
Anas Nashif55c3dde2021-09-16 08:54:19 -04003841 elif instance.status == 'skipped':
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003842 testcase["status"] = "skipped"
3843 testcase["reason"] = instance.reason
3844 testcases.append(testcase)
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003845
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003846 suites = [ {"testcases": testcases} ]
3847 report["testsuites"] = suites
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003848
3849 with open(filename, "wt") as json_file:
Anas Nashifcd1dccf2020-12-10 15:07:52 -05003850 json.dump(report, json_file, indent=4, separators=(',',':'))
Spoorthy Priya Yeraboluf8f220a2020-09-23 06:28:50 -07003851
Anas Nashifce2b4182020-03-24 14:40:28 -04003852 def get_testcase(self, identifier):
3853 results = []
3854 for _, tc in self.testcases.items():
3855 for case in tc.cases:
3856 if case == identifier:
3857 results.append(tc)
3858 return results
3859
Anas Nashifce2b4182020-03-24 14:40:28 -04003860class CoverageTool:
3861 """ Base class for every supported coverage tool
3862 """
3863
3864 def __init__(self):
Anas Nashiff6462a32020-03-29 19:02:51 -04003865 self.gcov_tool = None
3866 self.base_dir = None
Anas Nashifce2b4182020-03-24 14:40:28 -04003867
3868 @staticmethod
3869 def factory(tool):
3870 if tool == 'lcov':
Anas Nashiff6462a32020-03-29 19:02:51 -04003871 t = Lcov()
3872 elif tool == 'gcovr':
Marcin Niestroj2652dc72020-08-10 22:27:03 +02003873 t = Gcovr()
Anas Nashiff6462a32020-03-29 19:02:51 -04003874 else:
3875 logger.error("Unsupported coverage tool specified: {}".format(tool))
3876 return None
3877
Anas Nashif211ef412021-01-12 07:15:59 -05003878 logger.debug(f"Select {tool} as the coverage tool...")
Anas Nashiff6462a32020-03-29 19:02:51 -04003879 return t
Anas Nashifce2b4182020-03-24 14:40:28 -04003880
3881 @staticmethod
Kentaro Sugimotoe8ab9872021-07-01 16:44:52 +09003882 def retrieve_gcov_data(input_file):
3883 logger.debug("Working on %s" % input_file)
Anas Nashifce2b4182020-03-24 14:40:28 -04003884 extracted_coverage_info = {}
3885 capture_data = False
3886 capture_complete = False
Kentaro Sugimotoe8ab9872021-07-01 16:44:52 +09003887 with open(input_file, 'r') as fp:
Anas Nashifce2b4182020-03-24 14:40:28 -04003888 for line in fp.readlines():
3889 if re.search("GCOV_COVERAGE_DUMP_START", line):
3890 capture_data = True
3891 continue
3892 if re.search("GCOV_COVERAGE_DUMP_END", line):
3893 capture_complete = True
3894 break
3895 # Loop until the coverage data is found.
3896 if not capture_data:
3897 continue
3898 if line.startswith("*"):
3899 sp = line.split("<")
3900 if len(sp) > 1:
3901 # Remove the leading delimiter "*"
3902 file_name = sp[0][1:]
3903 # Remove the trailing new line char
3904 hex_dump = sp[1][:-1]
3905 else:
3906 continue
3907 else:
3908 continue
3909 extracted_coverage_info.update({file_name: hex_dump})
3910 if not capture_data:
3911 capture_complete = True
3912 return {'complete': capture_complete, 'data': extracted_coverage_info}
3913
3914 @staticmethod
3915 def create_gcda_files(extracted_coverage_info):
Anas Nashiff6462a32020-03-29 19:02:51 -04003916 logger.debug("Generating gcda files")
Anas Nashifce2b4182020-03-24 14:40:28 -04003917 for filename, hexdump_val in extracted_coverage_info.items():
3918 # if kobject_hash is given for coverage gcovr fails
3919 # hence skipping it problem only in gcovr v4.1
3920 if "kobject_hash" in filename:
3921 filename = (filename[:-4]) + "gcno"
3922 try:
3923 os.remove(filename)
3924 except Exception:
3925 pass
3926 continue
3927
3928 with open(filename, 'wb') as fp:
3929 fp.write(bytes.fromhex(hexdump_val))
3930
3931 def generate(self, outdir):
3932 for filename in glob.glob("%s/**/handler.log" % outdir, recursive=True):
3933 gcov_data = self.__class__.retrieve_gcov_data(filename)
3934 capture_complete = gcov_data['complete']
3935 extracted_coverage_info = gcov_data['data']
3936 if capture_complete:
3937 self.__class__.create_gcda_files(extracted_coverage_info)
3938 logger.debug("Gcov data captured: {}".format(filename))
3939 else:
3940 logger.error("Gcov data capture incomplete: {}".format(filename))
3941
3942 with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
3943 ret = self._generate(outdir, coveragelog)
3944 if ret == 0:
3945 logger.info("HTML report generated: {}".format(
3946 os.path.join(outdir, "coverage", "index.html")))
3947
3948
3949class Lcov(CoverageTool):
3950
3951 def __init__(self):
3952 super().__init__()
3953 self.ignores = []
3954
3955 def add_ignore_file(self, pattern):
3956 self.ignores.append('*' + pattern + '*')
3957
3958 def add_ignore_directory(self, pattern):
Marcin Niestrojfc674092020-08-10 23:22:11 +02003959 self.ignores.append('*/' + pattern + '/*')
Anas Nashifce2b4182020-03-24 14:40:28 -04003960
3961 def _generate(self, outdir, coveragelog):
3962 coveragefile = os.path.join(outdir, "coverage.info")
3963 ztestfile = os.path.join(outdir, "ztest.info")
Anas Nashif211ef412021-01-12 07:15:59 -05003964 cmd = ["lcov", "--gcov-tool", self.gcov_tool,
Anas Nashifce2b4182020-03-24 14:40:28 -04003965 "--capture", "--directory", outdir,
3966 "--rc", "lcov_branch_coverage=1",
Anas Nashif211ef412021-01-12 07:15:59 -05003967 "--output-file", coveragefile]
3968 cmd_str = " ".join(cmd)
3969 logger.debug(f"Running {cmd_str}...")
3970 subprocess.call(cmd, stdout=coveragelog)
Anas Nashifce2b4182020-03-24 14:40:28 -04003971 # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
3972 subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--extract",
3973 coveragefile,
Anas Nashiff6462a32020-03-29 19:02:51 -04003974 os.path.join(self.base_dir, "tests", "ztest", "*"),
Anas Nashifce2b4182020-03-24 14:40:28 -04003975 "--output-file", ztestfile,
3976 "--rc", "lcov_branch_coverage=1"], stdout=coveragelog)
3977
3978 if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
3979 subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--remove",
3980 ztestfile,
Anas Nashiff6462a32020-03-29 19:02:51 -04003981 os.path.join(self.base_dir, "tests/ztest/test/*"),
Anas Nashifce2b4182020-03-24 14:40:28 -04003982 "--output-file", ztestfile,
3983 "--rc", "lcov_branch_coverage=1"],
3984 stdout=coveragelog)
3985 files = [coveragefile, ztestfile]
3986 else:
3987 files = [coveragefile]
3988
3989 for i in self.ignores:
3990 subprocess.call(
3991 ["lcov", "--gcov-tool", self.gcov_tool, "--remove",
3992 coveragefile, i, "--output-file",
3993 coveragefile, "--rc", "lcov_branch_coverage=1"],
3994 stdout=coveragelog)
3995
3996 # The --ignore-errors source option is added to avoid it exiting due to
3997 # samples/application_development/external_lib/
3998 return subprocess.call(["genhtml", "--legend", "--branch-coverage",
3999 "--ignore-errors", "source",
4000 "-output-directory",
4001 os.path.join(outdir, "coverage")] + files,
4002 stdout=coveragelog)
4003
4004
4005class Gcovr(CoverageTool):
4006
4007 def __init__(self):
4008 super().__init__()
4009 self.ignores = []
4010
4011 def add_ignore_file(self, pattern):
4012 self.ignores.append('.*' + pattern + '.*')
4013
4014 def add_ignore_directory(self, pattern):
Marcin Niestrojfc674092020-08-10 23:22:11 +02004015 self.ignores.append(".*/" + pattern + '/.*')
Anas Nashifce2b4182020-03-24 14:40:28 -04004016
4017 @staticmethod
4018 def _interleave_list(prefix, list):
4019 tuple_list = [(prefix, item) for item in list]
4020 return [item for sublist in tuple_list for item in sublist]
4021
4022 def _generate(self, outdir, coveragelog):
4023 coveragefile = os.path.join(outdir, "coverage.json")
4024 ztestfile = os.path.join(outdir, "ztest.json")
4025
4026 excludes = Gcovr._interleave_list("-e", self.ignores)
4027
4028 # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
Anas Nashif211ef412021-01-12 07:15:59 -05004029 cmd = ["gcovr", "-r", self.base_dir, "--gcov-executable",
4030 self.gcov_tool, "-e", "tests/*"] + excludes + ["--json", "-o",
4031 coveragefile, outdir]
4032 cmd_str = " ".join(cmd)
4033 logger.debug(f"Running {cmd_str}...")
4034 subprocess.call(cmd, stdout=coveragelog)
Anas Nashifce2b4182020-03-24 14:40:28 -04004035
Anas Nashiff6462a32020-03-29 19:02:51 -04004036 subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable",
Anas Nashifce2b4182020-03-24 14:40:28 -04004037 self.gcov_tool, "-f", "tests/ztest", "-e",
4038 "tests/ztest/test/*", "--json", "-o", ztestfile,
4039 outdir], stdout=coveragelog)
4040
4041 if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
4042 files = [coveragefile, ztestfile]
4043 else:
4044 files = [coveragefile]
4045
4046 subdir = os.path.join(outdir, "coverage")
4047 os.makedirs(subdir, exist_ok=True)
4048
4049 tracefiles = self._interleave_list("--add-tracefile", files)
4050
Anas Nashiff6462a32020-03-29 19:02:51 -04004051 return subprocess.call(["gcovr", "-r", self.base_dir, "--html",
Anas Nashifce2b4182020-03-24 14:40:28 -04004052 "--html-details"] + tracefiles +
4053 ["-o", os.path.join(subdir, "index.html")],
4054 stdout=coveragelog)
Anas Nashifce2b4182020-03-24 14:40:28 -04004055
Dennis Rufferc714c782021-10-05 15:34:54 -07004056
Anas Nashif8305d1b2020-11-26 11:55:02 -05004057class DUT(object):
Anas Nashif531fe892020-09-11 13:56:33 -04004058 def __init__(self,
4059 id=None,
4060 serial=None,
Dennis Rufferc714c782021-10-05 15:34:54 -07004061 serial_baud=None,
Anas Nashif531fe892020-09-11 13:56:33 -04004062 platform=None,
4063 product=None,
4064 serial_pty=None,
Anas Nashif531fe892020-09-11 13:56:33 -04004065 connected=False,
4066 pre_script=None,
Anas Nashif8305d1b2020-11-26 11:55:02 -05004067 post_script=None,
4068 post_flash_script=None,
Anas Nashif531fe892020-09-11 13:56:33 -04004069 runner=None):
4070
4071 self.serial = serial
Dennis Rufferc714c782021-10-05 15:34:54 -07004072 self.serial_baud = 115200
4073 if serial_baud:
4074 self.serial_baud = serial_baud
Anas Nashif531fe892020-09-11 13:56:33 -04004075 self.platform = platform
4076 self.serial_pty = serial_pty
4077 self._counter = Value("i", 0)
4078 self._available = Value("i", 1)
4079 self.connected = connected
4080 self.pre_script = pre_script
4081 self.id = id
4082 self.product = product
4083 self.runner = runner
4084 self.fixtures = []
Anas Nashif8305d1b2020-11-26 11:55:02 -05004085 self.post_flash_script = post_flash_script
4086 self.post_script = post_script
4087 self.pre_script = pre_script
Anas Nashif531fe892020-09-11 13:56:33 -04004088 self.probe_id = None
4089 self.notes = None
Andy Ross098fce32021-03-02 05:13:07 -08004090 self.lock = Lock()
Anas Nashif531fe892020-09-11 13:56:33 -04004091 self.match = False
4092
4093
4094 @property
4095 def available(self):
4096 with self._available.get_lock():
4097 return self._available.value
4098
4099 @available.setter
4100 def available(self, value):
4101 with self._available.get_lock():
4102 self._available.value = value
4103
4104 @property
4105 def counter(self):
4106 with self._counter.get_lock():
4107 return self._counter.value
4108
4109 @counter.setter
4110 def counter(self, value):
4111 with self._counter.get_lock():
4112 self._counter.value = value
4113
Anas Nashif4fa82c42020-12-15 10:08:28 -05004114 def to_dict(self):
4115 d = {}
4116 exclude = ['_available', '_counter', 'match']
4117 v = vars(self)
4118 for k in v.keys():
4119 if k not in exclude and v[k]:
4120 d[k] = v[k]
4121 return d
4122
4123
Anas Nashif531fe892020-09-11 13:56:33 -04004124 def __repr__(self):
4125 return f"<{self.platform} ({self.product}) on {self.serial}>"
4126
4127class HardwareMap:
Anas Nashifc24bf6f2020-12-07 11:18:07 -05004128 schema_path = os.path.join(ZEPHYR_BASE, "scripts", "schemas", "twister", "hwmap-schema.yaml")
Anas Nashifce2b4182020-03-24 14:40:28 -04004129
4130 manufacturer = [
4131 'ARM',
4132 'SEGGER',
4133 'MBED',
4134 'STMicroelectronics',
4135 'Atmel Corp.',
4136 'Texas Instruments',
4137 'Silicon Labs',
4138 'NXP Semiconductors',
4139 'Microchip Technology Inc.',
4140 'FTDI',
4141 'Digilent'
4142 ]
4143
4144 runner_mapping = {
4145 'pyocd': [
4146 'DAPLink CMSIS-DAP',
4147 'MBED CMSIS-DAP'
4148 ],
4149 'jlink': [
4150 'J-Link',
4151 'J-Link OB'
4152 ],
4153 'openocd': [
Erwan Gouriou2339fa02020-07-07 17:15:22 +02004154 'STM32 STLink', '^XDS110.*', 'STLINK-V3'
Anas Nashifce2b4182020-03-24 14:40:28 -04004155 ],
4156 'dediprog': [
4157 'TTL232R-3V3',
4158 'MCP2200 USB Serial Port Emulator'
4159 ]
4160 }
4161
4162 def __init__(self):
4163 self.detected = []
Anas Nashif8305d1b2020-11-26 11:55:02 -05004164 self.duts = []
Anas Nashifce2b4182020-03-24 14:40:28 -04004165
Dennis Rufferc714c782021-10-05 15:34:54 -07004166 def add_device(self, serial, platform, pre_script, is_pty, baud=None):
4167 device = DUT(platform=platform, connected=True, pre_script=pre_script, serial_baud=baud)
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +03004168
4169 if is_pty:
Anas Nashif531fe892020-09-11 13:56:33 -04004170 device.serial_pty = serial
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +03004171 else:
Anas Nashif531fe892020-09-11 13:56:33 -04004172 device.serial = serial
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +03004173
Anas Nashif8305d1b2020-11-26 11:55:02 -05004174 self.duts.append(device)
Anas Nashifce2b4182020-03-24 14:40:28 -04004175
Anas Nashifad70a692020-11-26 09:19:41 -05004176 def load(self, map_file):
Anas Nashifce2b4182020-03-24 14:40:28 -04004177 hwm_schema = scl.yaml_load(self.schema_path)
Anas Nashif8305d1b2020-11-26 11:55:02 -05004178 duts = scl.yaml_load_verify(map_file, hwm_schema)
4179 for dut in duts:
4180 pre_script = dut.get('pre_script')
4181 post_script = dut.get('post_script')
4182 post_flash_script = dut.get('post_flash_script')
4183 platform = dut.get('platform')
4184 id = dut.get('id')
4185 runner = dut.get('runner')
4186 serial = dut.get('serial')
Dennis Rufferc714c782021-10-05 15:34:54 -07004187 baud = dut.get('baud', None)
Anas Nashif8305d1b2020-11-26 11:55:02 -05004188 product = dut.get('product')
Anas Nashifce954d42021-02-04 09:31:07 -05004189 fixtures = dut.get('fixtures', [])
Anas Nashif8305d1b2020-11-26 11:55:02 -05004190 new_dut = DUT(platform=platform,
4191 product=product,
4192 runner=runner,
4193 id=id,
4194 serial=serial,
Dennis Rufferc714c782021-10-05 15:34:54 -07004195 serial_baud=baud,
Anas Nashif8305d1b2020-11-26 11:55:02 -05004196 connected=serial is not None,
4197 pre_script=pre_script,
4198 post_script=post_script,
4199 post_flash_script=post_flash_script)
Anas Nashifce954d42021-02-04 09:31:07 -05004200 new_dut.fixtures = fixtures
Anas Nashif8305d1b2020-11-26 11:55:02 -05004201 new_dut.counter = 0
4202 self.duts.append(new_dut)
Anas Nashifce2b4182020-03-24 14:40:28 -04004203
Anas Nashifad70a692020-11-26 09:19:41 -05004204 def scan(self, persistent=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04004205 from serial.tools import list_ports
4206
Martí Bolívar07dce822020-04-13 16:50:51 -07004207 if persistent and platform.system() == 'Linux':
4208 # On Linux, /dev/serial/by-id provides symlinks to
4209 # '/dev/ttyACMx' nodes using names which are unique as
4210 # long as manufacturers fill out USB metadata nicely.
4211 #
4212 # This creates a map from '/dev/ttyACMx' device nodes
4213 # to '/dev/serial/by-id/usb-...' symlinks. The symlinks
4214 # go into the hardware map because they stay the same
4215 # even when the user unplugs / replugs the device.
4216 #
4217 # Some inexpensive USB/serial adapters don't result
4218 # in unique names here, though, so use of this feature
4219 # requires explicitly setting persistent=True.
4220 by_id = Path('/dev/serial/by-id')
4221 def readlink(link):
4222 return str((by_id / link).resolve())
4223
4224 persistent_map = {readlink(link): str(link)
4225 for link in by_id.iterdir()}
4226 else:
4227 persistent_map = {}
4228
Anas Nashifce2b4182020-03-24 14:40:28 -04004229 serial_devices = list_ports.comports()
4230 logger.info("Scanning connected hardware...")
4231 for d in serial_devices:
4232 if d.manufacturer in self.manufacturer:
4233
4234 # TI XDS110 can have multiple serial devices for a single board
4235 # assume endpoint 0 is the serial, skip all others
4236 if d.manufacturer == 'Texas Instruments' and not d.location.endswith('0'):
4237 continue
Anas Nashif8305d1b2020-11-26 11:55:02 -05004238 s_dev = DUT(platform="unknown",
Anas Nashif531fe892020-09-11 13:56:33 -04004239 id=d.serial_number,
4240 serial=persistent_map.get(d.device, d.device),
4241 product=d.product,
Anas Nashif0e64b872021-05-17 09:45:03 -04004242 runner='unknown',
4243 connected=True)
Anas Nashif531fe892020-09-11 13:56:33 -04004244
Anas Nashifce2b4182020-03-24 14:40:28 -04004245 for runner, _ in self.runner_mapping.items():
4246 products = self.runner_mapping.get(runner)
4247 if d.product in products:
Anas Nashif531fe892020-09-11 13:56:33 -04004248 s_dev.runner = runner
Anas Nashifce2b4182020-03-24 14:40:28 -04004249 continue
4250 # Try regex matching
4251 for p in products:
4252 if re.match(p, d.product):
Anas Nashif531fe892020-09-11 13:56:33 -04004253 s_dev.runner = runner
Anas Nashifce2b4182020-03-24 14:40:28 -04004254
Anas Nashif531fe892020-09-11 13:56:33 -04004255 s_dev.connected = True
Anas Nashifce2b4182020-03-24 14:40:28 -04004256 self.detected.append(s_dev)
4257 else:
4258 logger.warning("Unsupported device (%s): %s" % (d.manufacturer, d))
4259
Anas Nashifad70a692020-11-26 09:19:41 -05004260 def save(self, hwm_file):
Anas Nashifce2b4182020-03-24 14:40:28 -04004261 # use existing map
Anas Nashif4fa82c42020-12-15 10:08:28 -05004262 self.detected.sort(key=lambda x: x.serial or '')
Anas Nashifce2b4182020-03-24 14:40:28 -04004263 if os.path.exists(hwm_file):
4264 with open(hwm_file, 'r') as yaml_file:
Anas Nashifae61b7e2020-07-06 11:30:55 -04004265 hwm = yaml.load(yaml_file, Loader=SafeLoader)
Anas Nashif4fa82c42020-12-15 10:08:28 -05004266 if hwm:
4267 hwm.sort(key=lambda x: x['serial'] or '')
Øyvind Rønningstad4813f462020-07-01 16:49:38 +02004268
Anas Nashif4fa82c42020-12-15 10:08:28 -05004269 # disconnect everything
Anas Nashifce2b4182020-03-24 14:40:28 -04004270 for h in hwm:
Anas Nashif4fa82c42020-12-15 10:08:28 -05004271 h['connected'] = False
4272 h['serial'] = None
Anas Nashifce2b4182020-03-24 14:40:28 -04004273
Anas Nashif4fa82c42020-12-15 10:08:28 -05004274 for _detected in self.detected:
4275 for h in hwm:
4276 if _detected.id == h['id'] and _detected.product == h['product'] and not _detected.match:
4277 h['connected'] = True
4278 h['serial'] = _detected.serial
4279 _detected.match = True
4280
4281 new_duts = list(filter(lambda d: not d.match, self.detected))
4282 new = []
4283 for d in new_duts:
4284 new.append(d.to_dict())
4285
4286 if hwm:
4287 hwm = hwm + new
4288 else:
4289 hwm = new
Anas Nashifce2b4182020-03-24 14:40:28 -04004290
Anas Nashifce2b4182020-03-24 14:40:28 -04004291 with open(hwm_file, 'w') as yaml_file:
Anas Nashifae61b7e2020-07-06 11:30:55 -04004292 yaml.dump(hwm, yaml_file, Dumper=Dumper, default_flow_style=False)
Anas Nashifce2b4182020-03-24 14:40:28 -04004293
Anas Nashifad70a692020-11-26 09:19:41 -05004294 self.load(hwm_file)
4295 logger.info("Registered devices:")
4296 self.dump()
4297
Anas Nashifce2b4182020-03-24 14:40:28 -04004298 else:
4299 # create new file
Anas Nashifad70a692020-11-26 09:19:41 -05004300 dl = []
4301 for _connected in self.detected:
4302 platform = _connected.platform
4303 id = _connected.id
4304 runner = _connected.runner
4305 serial = _connected.serial
4306 product = _connected.product
4307 d = {
4308 'platform': platform,
4309 'id': id,
4310 'runner': runner,
4311 'serial': serial,
Anas Nashif0e64b872021-05-17 09:45:03 -04004312 'product': product,
4313 'connected': _connected.connected
Anas Nashifad70a692020-11-26 09:19:41 -05004314 }
4315 dl.append(d)
Anas Nashifce2b4182020-03-24 14:40:28 -04004316 with open(hwm_file, 'w') as yaml_file:
Anas Nashifad70a692020-11-26 09:19:41 -05004317 yaml.dump(dl, yaml_file, Dumper=Dumper, default_flow_style=False)
Anas Nashifce2b4182020-03-24 14:40:28 -04004318 logger.info("Detected devices:")
Anas Nashifad70a692020-11-26 09:19:41 -05004319 self.dump(detected=True)
Anas Nashifce2b4182020-03-24 14:40:28 -04004320
Anas Nashif531fe892020-09-11 13:56:33 -04004321 def dump(self, filtered=[], header=[], connected_only=False, detected=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04004322 print("")
4323 table = []
Anas Nashif531fe892020-09-11 13:56:33 -04004324 if detected:
4325 to_show = self.detected
4326 else:
Anas Nashif8305d1b2020-11-26 11:55:02 -05004327 to_show = self.duts
Anas Nashifad70a692020-11-26 09:19:41 -05004328
Anas Nashifce2b4182020-03-24 14:40:28 -04004329 if not header:
4330 header = ["Platform", "ID", "Serial device"]
Anas Nashif531fe892020-09-11 13:56:33 -04004331 for p in to_show:
4332 platform = p.platform
4333 connected = p.connected
Anas Nashifce2b4182020-03-24 14:40:28 -04004334 if filtered and platform not in filtered:
4335 continue
4336
4337 if not connected_only or connected:
Anas Nashif531fe892020-09-11 13:56:33 -04004338 table.append([platform, p.id, p.serial])
Anas Nashifce2b4182020-03-24 14:40:28 -04004339
4340 print(tabulate(table, headers=header, tablefmt="github"))