blob: d372d25baf003159c2cbbe0f8ab78db08b870258 [file] [log] [blame]
Anas Nashifce2b4182020-03-24 14:40:28 -04001#!/usr/bin/env python3
2# vim: set syntax=python ts=4 :
3#
4# Copyright (c) 2018 Intel Corporation
5# SPDX-License-Identifier: Apache-2.0
6
7import os
8import contextlib
9import string
10import mmap
11import sys
12import re
13import subprocess
14import select
15import shutil
16import shlex
17import signal
18import threading
19import concurrent.futures
20from collections import OrderedDict
21from threading import BoundedSemaphore
22import queue
23import time
24import csv
25import glob
26import concurrent
27import xml.etree.ElementTree as ET
28import logging
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +030029import pty
Anas Nashifce2b4182020-03-24 14:40:28 -040030from pathlib import Path
Martí Bolívar9861e5d2020-07-23 10:02:19 -070031import traceback
Anas Nashifce2b4182020-03-24 14:40:28 -040032from distutils.spawn import find_executable
33from colorama import Fore
Martí Bolívar9c92baa2020-07-08 14:43:07 -070034import pickle
Martí Bolívar07dce822020-04-13 16:50:51 -070035import platform
Anas Nashifae61b7e2020-07-06 11:30:55 -040036import yaml
37try:
38 # Use the C LibYAML parser if available, rather than the Python parser.
39 # It's much faster.
Anas Nashifae61b7e2020-07-06 11:30:55 -040040 from yaml import CSafeLoader as SafeLoader
41 from yaml import CDumper as Dumper
42except ImportError:
Martí Bolívard8698cb2020-07-08 14:55:14 -070043 from yaml import SafeLoader, Dumper
Anas Nashifce2b4182020-03-24 14:40:28 -040044
45try:
46 import serial
47except ImportError:
48 print("Install pyserial python module with pip to use --device-testing option.")
49
50try:
51 from tabulate import tabulate
52except ImportError:
53 print("Install tabulate python module with pip to use --device-testing option.")
54
Wentong Wu0d619ae2020-05-05 19:46:49 -040055try:
56 import psutil
57except ImportError:
Anas Nashif77946fa2020-05-21 18:19:01 -040058 print("Install psutil python module with pip to run in Qemu.")
Wentong Wu0d619ae2020-05-05 19:46:49 -040059
Anas Nashifce2b4182020-03-24 14:40:28 -040060ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
61if not ZEPHYR_BASE:
62 sys.exit("$ZEPHYR_BASE environment variable undefined")
63
Martí Bolívar9c92baa2020-07-08 14:43:07 -070064# This is needed to load edt.pickle files.
Anas Nashifce2b4182020-03-24 14:40:28 -040065sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts"))
Martí Bolívar469f53c2020-07-24 11:19:53 -070066import edtlib # pylint: disable=unused-import
Anas Nashifce2b4182020-03-24 14:40:28 -040067
68hw_map_local = threading.Lock()
69report_lock = threading.Lock()
70
71# Use this for internal comparisons; that's what canonicalization is
72# for. Don't use it when invoking other components of the build system
73# to avoid confusing and hard to trace inconsistencies in error messages
74# and logs, generated Makefiles, etc. compared to when users invoke these
75# components directly.
76# Note "normalization" is different from canonicalization, see os.path.
77canonical_zephyr_base = os.path.realpath(ZEPHYR_BASE)
78
79sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/"))
80
81from sanity_chk import scl
82from sanity_chk import expr_parser
83
Anas Nashifce2b4182020-03-24 14:40:28 -040084logger = logging.getLogger('sanitycheck')
85logger.setLevel(logging.DEBUG)
86
Anas Nashifce2b4182020-03-24 14:40:28 -040087pipeline = queue.LifoQueue()
88
89class CMakeCacheEntry:
90 '''Represents a CMake cache entry.
91
92 This class understands the type system in a CMakeCache.txt, and
93 converts the following cache types to Python types:
94
95 Cache Type Python type
96 ---------- -------------------------------------------
97 FILEPATH str
98 PATH str
99 STRING str OR list of str (if ';' is in the value)
100 BOOL bool
101 INTERNAL str OR list of str (if ';' is in the value)
102 ---------- -------------------------------------------
103 '''
104
105 # Regular expression for a cache entry.
106 #
107 # CMake variable names can include escape characters, allowing a
108 # wider set of names than is easy to match with a regular
109 # expression. To be permissive here, use a non-greedy match up to
110 # the first colon (':'). This breaks if the variable name has a
111 # colon inside, but it's good enough.
112 CACHE_ENTRY = re.compile(
113 r'''(?P<name>.*?) # name
114 :(?P<type>FILEPATH|PATH|STRING|BOOL|INTERNAL) # type
115 =(?P<value>.*) # value
116 ''', re.X)
117
118 @classmethod
119 def _to_bool(cls, val):
120 # Convert a CMake BOOL string into a Python bool.
121 #
122 # "True if the constant is 1, ON, YES, TRUE, Y, or a
123 # non-zero number. False if the constant is 0, OFF, NO,
124 # FALSE, N, IGNORE, NOTFOUND, the empty string, or ends in
125 # the suffix -NOTFOUND. Named boolean constants are
126 # case-insensitive. If the argument is not one of these
127 # constants, it is treated as a variable."
128 #
129 # https://cmake.org/cmake/help/v3.0/command/if.html
130 val = val.upper()
131 if val in ('ON', 'YES', 'TRUE', 'Y'):
132 return 1
133 elif val in ('OFF', 'NO', 'FALSE', 'N', 'IGNORE', 'NOTFOUND', ''):
134 return 0
135 elif val.endswith('-NOTFOUND'):
136 return 0
137 else:
138 try:
139 v = int(val)
140 return v != 0
141 except ValueError as exc:
142 raise ValueError('invalid bool {}'.format(val)) from exc
143
144 @classmethod
145 def from_line(cls, line, line_no):
146 # Comments can only occur at the beginning of a line.
147 # (The value of an entry could contain a comment character).
148 if line.startswith('//') or line.startswith('#'):
149 return None
150
151 # Whitespace-only lines do not contain cache entries.
152 if not line.strip():
153 return None
154
155 m = cls.CACHE_ENTRY.match(line)
156 if not m:
157 return None
158
159 name, type_, value = (m.group(g) for g in ('name', 'type', 'value'))
160 if type_ == 'BOOL':
161 try:
162 value = cls._to_bool(value)
163 except ValueError as exc:
164 args = exc.args + ('on line {}: {}'.format(line_no, line),)
165 raise ValueError(args) from exc
166 elif type_ in ['STRING', 'INTERNAL']:
167 # If the value is a CMake list (i.e. is a string which
168 # contains a ';'), convert to a Python list.
169 if ';' in value:
170 value = value.split(';')
171
172 return CMakeCacheEntry(name, value)
173
174 def __init__(self, name, value):
175 self.name = name
176 self.value = value
177
178 def __str__(self):
179 fmt = 'CMakeCacheEntry(name={}, value={})'
180 return fmt.format(self.name, self.value)
181
182
183class CMakeCache:
184 '''Parses and represents a CMake cache file.'''
185
186 @staticmethod
187 def from_file(cache_file):
188 return CMakeCache(cache_file)
189
190 def __init__(self, cache_file):
191 self.cache_file = cache_file
192 self.load(cache_file)
193
194 def load(self, cache_file):
195 entries = []
196 with open(cache_file, 'r') as cache:
197 for line_no, line in enumerate(cache):
198 entry = CMakeCacheEntry.from_line(line, line_no)
199 if entry:
200 entries.append(entry)
201 self._entries = OrderedDict((e.name, e) for e in entries)
202
203 def get(self, name, default=None):
204 entry = self._entries.get(name)
205 if entry is not None:
206 return entry.value
207 else:
208 return default
209
210 def get_list(self, name, default=None):
211 if default is None:
212 default = []
213 entry = self._entries.get(name)
214 if entry is not None:
215 value = entry.value
216 if isinstance(value, list):
217 return value
218 elif isinstance(value, str):
219 return [value] if value else []
220 else:
221 msg = 'invalid value {} type {}'
222 raise RuntimeError(msg.format(value, type(value)))
223 else:
224 return default
225
226 def __contains__(self, name):
227 return name in self._entries
228
229 def __getitem__(self, name):
230 return self._entries[name].value
231
232 def __setitem__(self, name, entry):
233 if not isinstance(entry, CMakeCacheEntry):
234 msg = 'improper type {} for value {}, expecting CMakeCacheEntry'
235 raise TypeError(msg.format(type(entry), entry))
236 self._entries[name] = entry
237
238 def __delitem__(self, name):
239 del self._entries[name]
240
241 def __iter__(self):
242 return iter(self._entries.values())
243
244
245class SanityCheckException(Exception):
246 pass
247
248
249class SanityRuntimeError(SanityCheckException):
250 pass
251
252
253class ConfigurationError(SanityCheckException):
254 def __init__(self, cfile, message):
255 SanityCheckException.__init__(self, cfile + ": " + message)
256
257
258class BuildError(SanityCheckException):
259 pass
260
261
262class ExecutionError(SanityCheckException):
263 pass
264
265
266class HarnessImporter:
267
268 def __init__(self, name):
269 sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/sanity_chk"))
270 module = __import__("harness")
271 if name:
272 my_class = getattr(module, name)
273 else:
274 my_class = getattr(module, "Test")
275
276 self.instance = my_class()
277
278
279class Handler:
280 def __init__(self, instance, type_str="build"):
281 """Constructor
282
283 """
284 self.lock = threading.Lock()
285
286 self.state = "waiting"
287 self.run = False
288 self.duration = 0
289 self.type_str = type_str
290
291 self.binary = None
292 self.pid_fn = None
293 self.call_make_run = False
294
295 self.name = instance.name
296 self.instance = instance
297 self.timeout = instance.testcase.timeout
298 self.sourcedir = instance.testcase.source_dir
299 self.build_dir = instance.build_dir
300 self.log = os.path.join(self.build_dir, "handler.log")
301 self.returncode = 0
302 self.set_state("running", self.duration)
303 self.generator = None
304 self.generator_cmd = None
305
306 self.args = []
307
308 def set_state(self, state, duration):
309 self.lock.acquire()
310 self.state = state
311 self.duration = duration
312 self.lock.release()
313
314 def get_state(self):
315 self.lock.acquire()
316 ret = (self.state, self.duration)
317 self.lock.release()
318 return ret
319
320 def record(self, harness):
321 if harness.recording:
322 filename = os.path.join(self.build_dir, "recording.csv")
323 with open(filename, "at") as csvfile:
324 cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep)
325 cw.writerow(harness.fieldnames)
326 for instance in harness.recording:
327 cw.writerow(instance)
328
329
330class BinaryHandler(Handler):
331 def __init__(self, instance, type_str):
332 """Constructor
333
334 @param instance Test Instance
335 """
336 super().__init__(instance, type_str)
337
338 self.terminated = False
339
340 # Tool options
341 self.valgrind = False
342 self.lsan = False
343 self.asan = False
Christian Taedcke3dbe9f22020-07-06 16:00:57 +0200344 self.ubsan = False
Anas Nashifce2b4182020-03-24 14:40:28 -0400345 self.coverage = False
346
347 def try_kill_process_by_pid(self):
348 if self.pid_fn:
349 pid = int(open(self.pid_fn).read())
350 os.unlink(self.pid_fn)
351 self.pid_fn = None # clear so we don't try to kill the binary twice
352 try:
353 os.kill(pid, signal.SIGTERM)
354 except ProcessLookupError:
355 pass
356
357 def terminate(self, proc):
358 # encapsulate terminate functionality so we do it consistently where ever
359 # we might want to terminate the proc. We need try_kill_process_by_pid
360 # because of both how newer ninja (1.6.0 or greater) and .NET / renode
361 # work. Newer ninja's don't seem to pass SIGTERM down to the children
362 # so we need to use try_kill_process_by_pid.
363 self.try_kill_process_by_pid()
364 proc.terminate()
Anas Nashif227392c2020-04-27 20:31:56 -0400365 # sleep for a while before attempting to kill
366 time.sleep(0.5)
367 proc.kill()
Anas Nashifce2b4182020-03-24 14:40:28 -0400368 self.terminated = True
369
370 def _output_reader(self, proc, harness):
371 log_out_fp = open(self.log, "wt")
372 for line in iter(proc.stdout.readline, b''):
373 logger.debug("OUTPUT: {0}".format(line.decode('utf-8').rstrip()))
374 log_out_fp.write(line.decode('utf-8'))
375 log_out_fp.flush()
376 harness.handle(line.decode('utf-8').rstrip())
377 if harness.state:
378 try:
379 # POSIX arch based ztests end on their own,
380 # so let's give it up to 100ms to do so
381 proc.wait(0.1)
382 except subprocess.TimeoutExpired:
383 self.terminate(proc)
384 break
385
386 log_out_fp.close()
387
388 def handle(self):
389
390 harness_name = self.instance.testcase.harness.capitalize()
391 harness_import = HarnessImporter(harness_name)
392 harness = harness_import.instance
393 harness.configure(self.instance)
394
395 if self.call_make_run:
396 command = [self.generator_cmd, "run"]
397 else:
398 command = [self.binary]
399
400 run_valgrind = False
401 if self.valgrind and shutil.which("valgrind"):
402 command = ["valgrind", "--error-exitcode=2",
403 "--leak-check=full",
404 "--suppressions=" + ZEPHYR_BASE + "/scripts/valgrind.supp",
405 "--log-file=" + self.build_dir + "/valgrind.log"
406 ] + command
407 run_valgrind = True
408
409 logger.debug("Spawning process: " +
410 " ".join(shlex.quote(word) for word in command) + os.linesep +
411 "in directory: " + self.build_dir)
412
413 start_time = time.time()
414
415 env = os.environ.copy()
416 if self.asan:
417 env["ASAN_OPTIONS"] = "log_path=stdout:" + \
418 env.get("ASAN_OPTIONS", "")
419 if not self.lsan:
420 env["ASAN_OPTIONS"] += "detect_leaks=0"
421
Christian Taedcke3dbe9f22020-07-06 16:00:57 +0200422 if self.ubsan:
423 env["UBSAN_OPTIONS"] = "log_path=stdout:halt_on_error=1:" + \
424 env.get("UBSAN_OPTIONS", "")
425
Anas Nashifce2b4182020-03-24 14:40:28 -0400426 with subprocess.Popen(command, stdout=subprocess.PIPE,
427 stderr=subprocess.PIPE, cwd=self.build_dir, env=env) as proc:
428 logger.debug("Spawning BinaryHandler Thread for %s" % self.name)
429 t = threading.Thread(target=self._output_reader, args=(proc, harness,), daemon=True)
430 t.start()
431 t.join(self.timeout)
432 if t.is_alive():
433 self.terminate(proc)
434 t.join()
435 proc.wait()
436 self.returncode = proc.returncode
437
438 handler_time = time.time() - start_time
439
440 if self.coverage:
441 subprocess.call(["GCOV_PREFIX=" + self.build_dir,
442 "gcov", self.sourcedir, "-b", "-s", self.build_dir], shell=True)
443
444 self.try_kill_process_by_pid()
445
446 # FIXME: This is needed when killing the simulator, the console is
447 # garbled and needs to be reset. Did not find a better way to do that.
448
449 subprocess.call(["stty", "sane"])
450 self.instance.results = harness.tests
451
452 if not self.terminated and self.returncode != 0:
453 # When a process is killed, the default handler returns 128 + SIGTERM
454 # so in that case the return code itself is not meaningful
455 self.set_state("failed", handler_time)
456 self.instance.reason = "Failed"
457 elif run_valgrind and self.returncode == 2:
458 self.set_state("failed", handler_time)
459 self.instance.reason = "Valgrind error"
460 elif harness.state:
461 self.set_state(harness.state, handler_time)
Anas Nashifb802af82020-04-26 21:57:38 -0400462 if harness.state == "failed":
463 self.instance.reason = "Failed"
Anas Nashifce2b4182020-03-24 14:40:28 -0400464 else:
465 self.set_state("timeout", handler_time)
466 self.instance.reason = "Timeout"
467
468 self.record(harness)
469
470
471class DeviceHandler(Handler):
472
473 def __init__(self, instance, type_str):
474 """Constructor
475
476 @param instance Test Instance
477 """
478 super().__init__(instance, type_str)
479
480 self.suite = None
Anas Nashifce2b4182020-03-24 14:40:28 -0400481
482 def monitor_serial(self, ser, halt_fileno, harness):
483 log_out_fp = open(self.log, "wt")
484
485 ser_fileno = ser.fileno()
486 readlist = [halt_fileno, ser_fileno]
487
488 while ser.isOpen():
489 readable, _, _ = select.select(readlist, [], [], self.timeout)
490
491 if halt_fileno in readable:
492 logger.debug('halted')
493 ser.close()
494 break
495 if ser_fileno not in readable:
496 continue # Timeout.
497
498 serial_line = None
499 try:
500 serial_line = ser.readline()
501 except TypeError:
502 pass
503 except serial.SerialException:
504 ser.close()
505 break
506
507 # Just because ser_fileno has data doesn't mean an entire line
508 # is available yet.
509 if serial_line:
510 sl = serial_line.decode('utf-8', 'ignore').lstrip()
511 logger.debug("DEVICE: {0}".format(sl.rstrip()))
512
513 log_out_fp.write(sl)
514 log_out_fp.flush()
515 harness.handle(sl.rstrip())
516
517 if harness.state:
518 ser.close()
519 break
520
521 log_out_fp.close()
522
Anas Nashif3b86f132020-05-21 10:35:33 -0400523 def device_is_available(self, instance):
524 device = instance.platform.name
525 fixture = instance.testcase.harness_config.get("fixture")
Anas Nashifce2b4182020-03-24 14:40:28 -0400526 for i in self.suite.connected_hardware:
Anas Nashif3b86f132020-05-21 10:35:33 -0400527 if fixture and fixture not in i.get('fixtures', []):
528 continue
Anas Nashife47866c2020-07-22 10:49:24 -0400529 if i['platform'] == device and i['available'] and (i['serial'] or i.get('serial_pty', None)):
Anas Nashifce2b4182020-03-24 14:40:28 -0400530 return True
531
532 return False
533
Anas Nashif3b86f132020-05-21 10:35:33 -0400534 def get_available_device(self, instance):
535 device = instance.platform.name
Anas Nashifce2b4182020-03-24 14:40:28 -0400536 for i in self.suite.connected_hardware:
Anas Nashife47866c2020-07-22 10:49:24 -0400537 if i['platform'] == device and i['available'] and (i['serial'] or i.get('serial_pty', None)):
Anas Nashifce2b4182020-03-24 14:40:28 -0400538 i['available'] = False
539 i['counter'] += 1
540 return i
541
542 return None
543
544 def make_device_available(self, serial):
545 with hw_map_local:
546 for i in self.suite.connected_hardware:
Anas Nashife47866c2020-07-22 10:49:24 -0400547 if i['serial'] == serial or i.get('serial_pty', None):
Anas Nashifce2b4182020-03-24 14:40:28 -0400548 i['available'] = True
549
550 @staticmethod
551 def run_custom_script(script, timeout):
552 with subprocess.Popen(script, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as proc:
553 try:
554 stdout, _ = proc.communicate(timeout=timeout)
555 logger.debug(stdout.decode())
556
557 except subprocess.TimeoutExpired:
558 proc.kill()
559 proc.communicate()
560 logger.error("{} timed out".format(script))
561
562 def handle(self):
563 out_state = "failed"
564
Anas Nashif3b86f132020-05-21 10:35:33 -0400565 while not self.device_is_available(self.instance):
Anas Nashifce2b4182020-03-24 14:40:28 -0400566 logger.debug("Waiting for device {} to become available".format(self.instance.platform.name))
567 time.sleep(1)
568
Anas Nashif3b86f132020-05-21 10:35:33 -0400569 hardware = self.get_available_device(self.instance)
Anas Nashif3b86f132020-05-21 10:35:33 -0400570 if hardware:
Øyvind Rønningstadf72aef12020-07-01 16:58:54 +0200571 runner = hardware.get('runner', None) or self.suite.west_runner
Anas Nashifce2b4182020-03-24 14:40:28 -0400572
Anas Nashife47866c2020-07-22 10:49:24 -0400573 serial_pty = hardware.get('serial_pty', None)
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300574 if serial_pty:
575 master, slave = pty.openpty()
576
577 try:
Andrei Emeltchenko9f7a9032020-09-02 11:43:07 +0300578 ser_pty_process = subprocess.Popen(re.split(',| ', serial_pty), stdout=master, stdin=master, stderr=master)
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300579 except subprocess.CalledProcessError as error:
580 logger.error("Failed to run subprocess {}, error {}".format(serial_pty, error.output))
581 return
582
583 serial_device = os.ttyname(slave)
584 else:
585 serial_device = hardware['serial']
586
587 logger.debug("Using serial device {}".format(serial_device))
Anas Nashifce2b4182020-03-24 14:40:28 -0400588
Øyvind Rønningstadf72aef12020-07-01 16:58:54 +0200589 if (self.suite.west_flash is not None) or runner:
590 command = ["west", "flash", "--skip-rebuild", "-d", self.build_dir]
591 command_extra_args = []
592
593 # There are three ways this option is used.
594 # 1) bare: --west-flash
595 # This results in options.west_flash == []
596 # 2) with a value: --west-flash="--board-id=42"
597 # This results in options.west_flash == "--board-id=42"
598 # 3) Multiple values: --west-flash="--board-id=42,--erase"
599 # This results in options.west_flash == "--board-id=42 --erase"
600 if self.suite.west_flash and self.suite.west_flash != []:
601 command_extra_args.extend(self.suite.west_flash.split(','))
602
603 if runner:
604 command.append("--runner")
605 command.append(runner)
606
607 board_id = hardware.get("probe_id", hardware.get("id", None))
608 product = hardware.get("product", None)
609 if board_id is not None:
610 if runner == "pyocd":
611 command_extra_args.append("--board-id")
612 command_extra_args.append(board_id)
613 elif runner == "nrfjprog":
614 command_extra_args.append("--snr")
615 command_extra_args.append(board_id)
616 elif runner == "openocd" and product == "STM32 STLink":
617 command_extra_args.append("--cmd-pre-init")
618 command_extra_args.append("hla_serial %s" % (board_id))
619 elif runner == "openocd" and product == "STLINK-V3":
620 command_extra_args.append("--cmd-pre-init")
621 command_extra_args.append("hla_serial %s" % (board_id))
622 elif runner == "openocd" and product == "EDBG CMSIS-DAP":
623 command_extra_args.append("--cmd-pre-init")
624 command_extra_args.append("cmsis_dap_serial %s" % (board_id))
625 elif runner == "jlink":
626 command.append("--tool-opt=-SelectEmuBySN %s" % (board_id))
627
628 if command_extra_args != []:
629 command.append('--')
630 command.extend(command_extra_args)
631 else:
632 command = [self.generator_cmd, "-C", self.build_dir, "flash"]
633
Watson Zeng3b43d942020-08-25 15:22:39 +0800634 pre_script = hardware.get('pre_script')
635 post_flash_script = hardware.get('post_flash_script')
636 post_script = hardware.get('post_script')
637
638 if pre_script:
639 self.run_custom_script(pre_script, 30)
640
Anas Nashifce2b4182020-03-24 14:40:28 -0400641 try:
642 ser = serial.Serial(
643 serial_device,
644 baudrate=115200,
645 parity=serial.PARITY_NONE,
646 stopbits=serial.STOPBITS_ONE,
647 bytesize=serial.EIGHTBITS,
648 timeout=self.timeout
649 )
650 except serial.SerialException as e:
651 self.set_state("failed", 0)
652 self.instance.reason = "Failed"
653 logger.error("Serial device error: %s" % (str(e)))
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300654
655 if serial_pty:
656 ser_pty_process.terminate()
657 outs, errs = ser_pty_process.communicate()
658 logger.debug("Process {} terminated outs: {} errs {}".format(serial_pty, outs, errs))
659
Anas Nashifce2b4182020-03-24 14:40:28 -0400660 self.make_device_available(serial_device)
661 return
662
663 ser.flush()
664
665 harness_name = self.instance.testcase.harness.capitalize()
666 harness_import = HarnessImporter(harness_name)
667 harness = harness_import.instance
668 harness.configure(self.instance)
669 read_pipe, write_pipe = os.pipe()
670 start_time = time.time()
671
Anas Nashifce2b4182020-03-24 14:40:28 -0400672 t = threading.Thread(target=self.monitor_serial, daemon=True,
673 args=(ser, read_pipe, harness))
674 t.start()
675
676 d_log = "{}/device.log".format(self.instance.build_dir)
677 logger.debug('Flash command: %s', command)
678 try:
679 stdout = stderr = None
680 with subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) as proc:
681 try:
682 (stdout, stderr) = proc.communicate(timeout=30)
683 logger.debug(stdout.decode())
684
685 if proc.returncode != 0:
686 self.instance.reason = "Device issue (Flash?)"
687 with open(d_log, "w") as dlog_fp:
688 dlog_fp.write(stderr.decode())
689 except subprocess.TimeoutExpired:
690 proc.kill()
691 (stdout, stderr) = proc.communicate()
692 self.instance.reason = "Device issue (Timeout)"
693
694 with open(d_log, "w") as dlog_fp:
695 dlog_fp.write(stderr.decode())
696
697 except subprocess.CalledProcessError:
698 os.write(write_pipe, b'x') # halt the thread
699
700 if post_flash_script:
701 self.run_custom_script(post_flash_script, 30)
702
Anas Nashifce2b4182020-03-24 14:40:28 -0400703 t.join(self.timeout)
704 if t.is_alive():
705 logger.debug("Timed out while monitoring serial output on {}".format(self.instance.platform.name))
706 out_state = "timeout"
707
708 if ser.isOpen():
709 ser.close()
710
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +0300711 if serial_pty:
712 ser_pty_process.terminate()
713 outs, errs = ser_pty_process.communicate()
714 logger.debug("Process {} terminated outs: {} errs {}".format(serial_pty, outs, errs))
715
Anas Nashifce2b4182020-03-24 14:40:28 -0400716 os.close(write_pipe)
717 os.close(read_pipe)
718
719 handler_time = time.time() - start_time
720
721 if out_state == "timeout":
722 for c in self.instance.testcase.cases:
723 if c not in harness.tests:
724 harness.tests[c] = "BLOCK"
725
726 self.instance.reason = "Timeout"
727
728 self.instance.results = harness.tests
729
730 if harness.state:
731 self.set_state(harness.state, handler_time)
732 if harness.state == "failed":
733 self.instance.reason = "Failed"
734 else:
735 self.set_state(out_state, handler_time)
736
737 if post_script:
738 self.run_custom_script(post_script, 30)
739
740 self.make_device_available(serial_device)
741
742 self.record(harness)
743
744
745class QEMUHandler(Handler):
746 """Spawns a thread to monitor QEMU output from pipes
747
748 We pass QEMU_PIPE to 'make run' and monitor the pipes for output.
749 We need to do this as once qemu starts, it runs forever until killed.
750 Test cases emit special messages to the console as they run, we check
751 for these to collect whether the test passed or failed.
752 """
753
754 def __init__(self, instance, type_str):
755 """Constructor
756
757 @param instance Test instance
758 """
759
760 super().__init__(instance, type_str)
761 self.fifo_fn = os.path.join(instance.build_dir, "qemu-fifo")
762
763 self.pid_fn = os.path.join(instance.build_dir, "qemu.pid")
764
Daniel Leungfaae15d2020-08-18 10:13:35 -0700765 if "ignore_qemu_crash" in instance.testcase.tags:
766 self.ignore_qemu_crash = True
767 self.ignore_unexpected_eof = True
768 else:
769 self.ignore_qemu_crash = False
770 self.ignore_unexpected_eof = False
771
Anas Nashifce2b4182020-03-24 14:40:28 -0400772 @staticmethod
Wentong Wu0d619ae2020-05-05 19:46:49 -0400773 def _get_cpu_time(pid):
774 """get process CPU time.
775
776 The guest virtual time in QEMU icount mode isn't host time and
777 it's maintained by counting guest instructions, so we use QEMU
778 process exection time to mostly simulate the time of guest OS.
779 """
780 proc = psutil.Process(pid)
781 cpu_time = proc.cpu_times()
782 return cpu_time.user + cpu_time.system
783
784 @staticmethod
Daniel Leungfaae15d2020-08-18 10:13:35 -0700785 def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results, harness,
786 ignore_unexpected_eof=False):
Anas Nashifce2b4182020-03-24 14:40:28 -0400787 fifo_in = fifo_fn + ".in"
788 fifo_out = fifo_fn + ".out"
789
790 # These in/out nodes are named from QEMU's perspective, not ours
791 if os.path.exists(fifo_in):
792 os.unlink(fifo_in)
793 os.mkfifo(fifo_in)
794 if os.path.exists(fifo_out):
795 os.unlink(fifo_out)
796 os.mkfifo(fifo_out)
797
798 # We don't do anything with out_fp but we need to open it for
799 # writing so that QEMU doesn't block, due to the way pipes work
800 out_fp = open(fifo_in, "wb")
801 # Disable internal buffering, we don't
802 # want read() or poll() to ever block if there is data in there
803 in_fp = open(fifo_out, "rb", buffering=0)
804 log_out_fp = open(logfile, "wt")
805
806 start_time = time.time()
807 timeout_time = start_time + timeout
808 p = select.poll()
809 p.register(in_fp, select.POLLIN)
810 out_state = None
811
812 line = ""
813 timeout_extended = False
Wentong Wu0d619ae2020-05-05 19:46:49 -0400814
815 pid = 0
816 if os.path.exists(pid_fn):
817 pid = int(open(pid_fn).read())
818
Anas Nashifce2b4182020-03-24 14:40:28 -0400819 while True:
820 this_timeout = int((timeout_time - time.time()) * 1000)
821 if this_timeout < 0 or not p.poll(this_timeout):
Wentong Wu517633c2020-07-24 21:13:01 +0800822 try:
823 if pid and this_timeout > 0:
824 #there's possibility we polled nothing because
825 #of not enough CPU time scheduled by host for
826 #QEMU process during p.poll(this_timeout)
827 cpu_time = QEMUHandler._get_cpu_time(pid)
828 if cpu_time < timeout and not out_state:
829 timeout_time = time.time() + (timeout - cpu_time)
830 continue
831 except ProcessLookupError:
832 out_state = "failed"
833 break
Wentong Wu0d619ae2020-05-05 19:46:49 -0400834
Anas Nashifce2b4182020-03-24 14:40:28 -0400835 if not out_state:
836 out_state = "timeout"
837 break
838
Wentong Wu0d619ae2020-05-05 19:46:49 -0400839 if pid == 0 and os.path.exists(pid_fn):
840 pid = int(open(pid_fn).read())
841
Anas Nashifce2b4182020-03-24 14:40:28 -0400842 try:
843 c = in_fp.read(1).decode("utf-8")
844 except UnicodeDecodeError:
845 # Test is writing something weird, fail
846 out_state = "unexpected byte"
847 break
848
849 if c == "":
850 # EOF, this shouldn't happen unless QEMU crashes
Daniel Leungfaae15d2020-08-18 10:13:35 -0700851 if not ignore_unexpected_eof:
852 out_state = "unexpected eof"
Anas Nashifce2b4182020-03-24 14:40:28 -0400853 break
854 line = line + c
855 if c != "\n":
856 continue
857
858 # line contains a full line of data output from QEMU
859 log_out_fp.write(line)
860 log_out_fp.flush()
861 line = line.strip()
862 logger.debug("QEMU: %s" % line)
863
864 harness.handle(line)
865 if harness.state:
866 # if we have registered a fail make sure the state is not
867 # overridden by a false success message coming from the
868 # testsuite
Anas Nashif869ca052020-07-07 14:29:07 -0400869 if out_state not in ['failed', 'unexpected eof', 'unexpected byte']:
Anas Nashifce2b4182020-03-24 14:40:28 -0400870 out_state = harness.state
871
872 # if we get some state, that means test is doing well, we reset
873 # the timeout and wait for 2 more seconds to catch anything
874 # printed late. We wait much longer if code
875 # coverage is enabled since dumping this information can
876 # take some time.
877 if not timeout_extended or harness.capture_coverage:
878 timeout_extended = True
879 if harness.capture_coverage:
880 timeout_time = time.time() + 30
881 else:
882 timeout_time = time.time() + 2
883 line = ""
884
885 handler.record(harness)
886
887 handler_time = time.time() - start_time
888 logger.debug("QEMU complete (%s) after %f seconds" %
889 (out_state, handler_time))
Anas Nashif869ca052020-07-07 14:29:07 -0400890
Anas Nashifce2b4182020-03-24 14:40:28 -0400891 if out_state == "timeout":
892 handler.instance.reason = "Timeout"
Anas Nashif06052922020-07-15 22:44:24 -0400893 handler.set_state("failed", handler_time)
Anas Nashifce2b4182020-03-24 14:40:28 -0400894 elif out_state == "failed":
895 handler.instance.reason = "Failed"
Anas Nashif869ca052020-07-07 14:29:07 -0400896 handler.set_state("failed", handler_time)
Anas Nashif06052922020-07-15 22:44:24 -0400897 elif out_state in ['unexpected eof', 'unexpected byte']:
Anas Nashif869ca052020-07-07 14:29:07 -0400898 handler.instance.reason = out_state
Anas Nashif06052922020-07-15 22:44:24 -0400899 handler.set_state("failed", handler_time)
900 else:
901 handler.set_state(out_state, handler_time)
Anas Nashifce2b4182020-03-24 14:40:28 -0400902
903 log_out_fp.close()
904 out_fp.close()
905 in_fp.close()
Wentong Wu0d619ae2020-05-05 19:46:49 -0400906 if pid:
Anas Nashifce2b4182020-03-24 14:40:28 -0400907 try:
908 if pid:
909 os.kill(pid, signal.SIGTERM)
910 except ProcessLookupError:
911 # Oh well, as long as it's dead! User probably sent Ctrl-C
912 pass
913
914 os.unlink(fifo_in)
915 os.unlink(fifo_out)
916
917 def handle(self):
918 self.results = {}
919 self.run = True
920
921 # We pass this to QEMU which looks for fifos with .in and .out
922 # suffixes.
Anas Nashifce2b4182020-03-24 14:40:28 -0400923
Anas Nashifc1c10992020-09-10 07:36:00 -0400924 self.fifo_fn = os.path.join(self.instance.build_dir, "qemu-fifo")
Anas Nashifce2b4182020-03-24 14:40:28 -0400925 self.pid_fn = os.path.join(self.instance.build_dir, "qemu.pid")
Anas Nashifc1c10992020-09-10 07:36:00 -0400926
Anas Nashifce2b4182020-03-24 14:40:28 -0400927 if os.path.exists(self.pid_fn):
928 os.unlink(self.pid_fn)
929
930 self.log_fn = self.log
931
932 harness_import = HarnessImporter(self.instance.testcase.harness.capitalize())
933 harness = harness_import.instance
934 harness.configure(self.instance)
935 self.thread = threading.Thread(name=self.name, target=QEMUHandler._thread,
936 args=(self, self.timeout, self.build_dir,
937 self.log_fn, self.fifo_fn,
Daniel Leungfaae15d2020-08-18 10:13:35 -0700938 self.pid_fn, self.results, harness,
939 self.ignore_unexpected_eof))
Anas Nashifce2b4182020-03-24 14:40:28 -0400940
941 self.instance.results = harness.tests
942 self.thread.daemon = True
943 logger.debug("Spawning QEMUHandler Thread for %s" % self.name)
944 self.thread.start()
945 subprocess.call(["stty", "sane"])
946
947 logger.debug("Running %s (%s)" % (self.name, self.type_str))
948 command = [self.generator_cmd]
949 command += ["-C", self.build_dir, "run"]
950
Anas Nashiffdc02b62020-09-09 19:11:19 -0400951 is_timeout = False
952
Anas Nashifce2b4182020-03-24 14:40:28 -0400953 with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.build_dir) as proc:
954 logger.debug("Spawning QEMUHandler Thread for %s" % self.name)
Wentong Wu7ec57b42020-05-05 19:19:18 -0400955 try:
956 proc.wait(self.timeout)
957 except subprocess.TimeoutExpired:
Anas Nashiffdc02b62020-09-09 19:11:19 -0400958 # sometimes QEMU can't handle SIGTERM signal correctly
959 # in that case kill -9 QEMU process directly and leave
Anas Nashifc1c10992020-09-10 07:36:00 -0400960 # sanitycheck to judge testing result by console output
Anas Nashiffdc02b62020-09-09 19:11:19 -0400961
962 is_timeout = True
Wentong Wu7ec57b42020-05-05 19:19:18 -0400963 if os.path.exists(self.pid_fn):
964 qemu_pid = int(open(self.pid_fn).read())
965 try:
966 os.kill(qemu_pid, signal.SIGKILL)
967 except ProcessLookupError:
968 pass
969 proc.wait()
Wentong Wu6fae53c2020-06-24 22:55:59 +0800970 if harness.state == "passed":
971 self.returncode = 0
972 else:
973 self.returncode = proc.returncode
Wentong Wu7ec57b42020-05-05 19:19:18 -0400974 else:
975 proc.terminate()
976 proc.kill()
977 self.returncode = proc.returncode
978 else:
Daniel Leung5b1b4a32020-08-18 10:10:36 -0700979 logger.debug(f"No timeout, return code from qemu: {proc.returncode}")
Wentong Wu7ec57b42020-05-05 19:19:18 -0400980 self.returncode = proc.returncode
981
Daniel Leung5b1b4a32020-08-18 10:10:36 -0700982 # Need to wait for harness to finish processing
983 # output from QEMU. Otherwise it might miss some
984 # error messages.
985 self.thread.join()
986
Wentong Wu7ec57b42020-05-05 19:19:18 -0400987 if os.path.exists(self.pid_fn):
988 os.unlink(self.pid_fn)
Anas Nashifce2b4182020-03-24 14:40:28 -0400989
Anas Nashif869ca052020-07-07 14:29:07 -0400990 logger.debug(f"return code from qemu: {self.returncode}")
991
Daniel Leungfaae15d2020-08-18 10:13:35 -0700992 if (self.returncode != 0 and not self.ignore_qemu_crash) or not harness.state:
Anas Nashifce2b4182020-03-24 14:40:28 -0400993 self.set_state("failed", 0)
Anas Nashiffdc02b62020-09-09 19:11:19 -0400994 if is_timeout:
995 self.instance.reason = "Timeout"
996 else:
997 self.instance.reason = "Exited with {}".format(self.returncode)
Anas Nashifce2b4182020-03-24 14:40:28 -0400998
999 def get_fifo(self):
1000 return self.fifo_fn
1001
1002
1003class SizeCalculator:
1004 alloc_sections = [
1005 "bss",
1006 "noinit",
1007 "app_bss",
1008 "app_noinit",
1009 "ccm_bss",
1010 "ccm_noinit"
1011 ]
1012
1013 rw_sections = [
1014 "datas",
1015 "initlevel",
1016 "exceptions",
1017 "initshell",
Andrew Boie45979da2020-05-23 14:38:39 -07001018 "_static_thread_data_area",
1019 "k_timer_area",
1020 "k_mem_slab_area",
1021 "k_mem_pool_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001022 "sw_isr_table",
Andrew Boie45979da2020-05-23 14:38:39 -07001023 "k_sem_area",
1024 "k_mutex_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001025 "app_shmem_regions",
1026 "_k_fifo_area",
1027 "_k_lifo_area",
Andrew Boie45979da2020-05-23 14:38:39 -07001028 "k_stack_area",
1029 "k_msgq_area",
1030 "k_mbox_area",
1031 "k_pipe_area",
Daniel Leung203556c2020-08-24 12:40:26 -07001032 "net_if_area",
1033 "net_if_dev_area",
1034 "net_l2_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001035 "net_l2_data",
Andrew Boie45979da2020-05-23 14:38:39 -07001036 "k_queue_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001037 "_net_buf_pool_area",
1038 "app_datas",
1039 "kobject_data",
1040 "mmu_tables",
1041 "app_pad",
1042 "priv_stacks",
1043 "ccm_data",
1044 "usb_descriptor",
1045 "usb_data", "usb_bos_desc",
Jukka Rissanen420b1952020-04-01 12:47:53 +03001046 "uart_mux",
Anas Nashifce2b4182020-03-24 14:40:28 -04001047 'log_backends_sections',
1048 'log_dynamic_sections',
1049 'log_const_sections',
1050 "app_smem",
1051 'shell_root_cmds_sections',
1052 'log_const_sections',
1053 "font_entry_sections",
1054 "priv_stacks_noinit",
1055 "_GCOV_BSS_SECTION_NAME",
1056 "gcov",
Daniel Leung203556c2020-08-24 12:40:26 -07001057 "nocache",
1058 "devices",
1059 "k_heap_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001060 ]
1061
1062 # These get copied into RAM only on non-XIP
1063 ro_sections = [
1064 "rom_start",
1065 "text",
1066 "ctors",
1067 "init_array",
1068 "reset",
Andrew Boie45979da2020-05-23 14:38:39 -07001069 "z_object_assignment_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001070 "rodata",
Anas Nashifce2b4182020-03-24 14:40:28 -04001071 "net_l2",
1072 "vector",
1073 "sw_isr_table",
Andrew Boie45979da2020-05-23 14:38:39 -07001074 "settings_handler_static_area",
Daniel Leung203556c2020-08-24 12:40:26 -07001075 "bt_l2cap_fixed_chan_area",
1076 "bt_l2cap_br_fixed_chan_area",
1077 "bt_gatt_service_static_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001078 "vectors",
Andrew Boie45979da2020-05-23 14:38:39 -07001079 "net_socket_register_area",
1080 "net_ppp_proto",
1081 "shell_area",
1082 "tracing_backend_area",
Daniel Leung203556c2020-08-24 12:40:26 -07001083 "ppp_protocol_handler_area",
Anas Nashifce2b4182020-03-24 14:40:28 -04001084 ]
1085
1086 def __init__(self, filename, extra_sections):
1087 """Constructor
1088
1089 @param filename Path to the output binary
1090 The <filename> is parsed by objdump to determine section sizes
1091 """
1092 # Make sure this is an ELF binary
1093 with open(filename, "rb") as f:
1094 magic = f.read(4)
1095
1096 try:
1097 if magic != b'\x7fELF':
1098 raise SanityRuntimeError("%s is not an ELF binary" % filename)
1099 except Exception as e:
1100 print(str(e))
1101 sys.exit(2)
1102
1103 # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
1104 # GREP can not be used as it returns an error if the symbol is not
1105 # found.
1106 is_xip_command = "nm " + filename + \
1107 " | awk '/CONFIG_XIP/ { print $3 }'"
1108 is_xip_output = subprocess.check_output(
1109 is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
1110 "utf-8").strip()
1111 try:
1112 if is_xip_output.endswith("no symbols"):
1113 raise SanityRuntimeError("%s has no symbol information" % filename)
1114 except Exception as e:
1115 print(str(e))
1116 sys.exit(2)
1117
1118 self.is_xip = (len(is_xip_output) != 0)
1119
1120 self.filename = filename
1121 self.sections = []
1122 self.rom_size = 0
1123 self.ram_size = 0
1124 self.extra_sections = extra_sections
1125
1126 self._calculate_sizes()
1127
1128 def get_ram_size(self):
1129 """Get the amount of RAM the application will use up on the device
1130
1131 @return amount of RAM, in bytes
1132 """
1133 return self.ram_size
1134
1135 def get_rom_size(self):
1136 """Get the size of the data that this application uses on device's flash
1137
1138 @return amount of ROM, in bytes
1139 """
1140 return self.rom_size
1141
1142 def unrecognized_sections(self):
1143 """Get a list of sections inside the binary that weren't recognized
1144
1145 @return list of unrecognized section names
1146 """
1147 slist = []
1148 for v in self.sections:
1149 if not v["recognized"]:
1150 slist.append(v["name"])
1151 return slist
1152
1153 def _calculate_sizes(self):
1154 """ Calculate RAM and ROM usage by section """
1155 objdump_command = "objdump -h " + self.filename
1156 objdump_output = subprocess.check_output(
1157 objdump_command, shell=True).decode("utf-8").splitlines()
1158
1159 for line in objdump_output:
1160 words = line.split()
1161
1162 if not words: # Skip lines that are too short
1163 continue
1164
1165 index = words[0]
1166 if not index[0].isdigit(): # Skip lines that do not start
1167 continue # with a digit
1168
1169 name = words[1] # Skip lines with section names
1170 if name[0] == '.': # starting with '.'
1171 continue
1172
1173 # TODO this doesn't actually reflect the size in flash or RAM as
1174 # it doesn't include linker-imposed padding between sections.
1175 # It is close though.
1176 size = int(words[2], 16)
1177 if size == 0:
1178 continue
1179
1180 load_addr = int(words[4], 16)
1181 virt_addr = int(words[3], 16)
1182
1183 # Add section to memory use totals (for both non-XIP and XIP scenarios)
1184 # Unrecognized section names are not included in the calculations.
1185 recognized = True
1186 if name in SizeCalculator.alloc_sections:
1187 self.ram_size += size
1188 stype = "alloc"
1189 elif name in SizeCalculator.rw_sections:
1190 self.ram_size += size
1191 self.rom_size += size
1192 stype = "rw"
1193 elif name in SizeCalculator.ro_sections:
1194 self.rom_size += size
1195 if not self.is_xip:
1196 self.ram_size += size
1197 stype = "ro"
1198 else:
1199 stype = "unknown"
1200 if name not in self.extra_sections:
1201 recognized = False
1202
1203 self.sections.append({"name": name, "load_addr": load_addr,
1204 "size": size, "virt_addr": virt_addr,
1205 "type": stype, "recognized": recognized})
1206
1207
1208
1209class SanityConfigParser:
1210 """Class to read test case files with semantic checking
1211 """
1212
1213 def __init__(self, filename, schema):
1214 """Instantiate a new SanityConfigParser object
1215
1216 @param filename Source .yaml file to read
1217 """
1218 self.data = {}
1219 self.schema = schema
1220 self.filename = filename
1221 self.tests = {}
1222 self.common = {}
1223
1224 def load(self):
1225 self.data = scl.yaml_load_verify(self.filename, self.schema)
1226
1227 if 'tests' in self.data:
1228 self.tests = self.data['tests']
1229 if 'common' in self.data:
1230 self.common = self.data['common']
1231
1232 def _cast_value(self, value, typestr):
1233 if isinstance(value, str):
1234 v = value.strip()
1235 if typestr == "str":
1236 return v
1237
1238 elif typestr == "float":
1239 return float(value)
1240
1241 elif typestr == "int":
1242 return int(value)
1243
1244 elif typestr == "bool":
1245 return value
1246
1247 elif typestr.startswith("list") and isinstance(value, list):
1248 return value
1249 elif typestr.startswith("list") and isinstance(value, str):
1250 vs = v.split()
1251 if len(typestr) > 4 and typestr[4] == ":":
1252 return [self._cast_value(vsi, typestr[5:]) for vsi in vs]
1253 else:
1254 return vs
1255
1256 elif typestr.startswith("set"):
1257 vs = v.split()
1258 if len(typestr) > 3 and typestr[3] == ":":
1259 return {self._cast_value(vsi, typestr[4:]) for vsi in vs}
1260 else:
1261 return set(vs)
1262
1263 elif typestr.startswith("map"):
1264 return value
1265 else:
1266 raise ConfigurationError(
1267 self.filename, "unknown type '%s'" % value)
1268
1269 def get_test(self, name, valid_keys):
1270 """Get a dictionary representing the keys/values within a test
1271
1272 @param name The test in the .yaml file to retrieve data from
1273 @param valid_keys A dictionary representing the intended semantics
1274 for this test. Each key in this dictionary is a key that could
1275 be specified, if a key is given in the .yaml file which isn't in
1276 here, it will generate an error. Each value in this dictionary
1277 is another dictionary containing metadata:
1278
1279 "default" - Default value if not given
1280 "type" - Data type to convert the text value to. Simple types
1281 supported are "str", "float", "int", "bool" which will get
1282 converted to respective Python data types. "set" and "list"
1283 may also be specified which will split the value by
1284 whitespace (but keep the elements as strings). finally,
1285 "list:<type>" and "set:<type>" may be given which will
1286 perform a type conversion after splitting the value up.
1287 "required" - If true, raise an error if not defined. If false
1288 and "default" isn't specified, a type conversion will be
1289 done on an empty string
1290 @return A dictionary containing the test key-value pairs with
1291 type conversion and default values filled in per valid_keys
1292 """
1293
1294 d = {}
1295 for k, v in self.common.items():
1296 d[k] = v
1297
1298 for k, v in self.tests[name].items():
Anas Nashifce2b4182020-03-24 14:40:28 -04001299 if k in d:
1300 if isinstance(d[k], str):
1301 # By default, we just concatenate string values of keys
1302 # which appear both in "common" and per-test sections,
1303 # but some keys are handled in adhoc way based on their
1304 # semantics.
1305 if k == "filter":
1306 d[k] = "(%s) and (%s)" % (d[k], v)
1307 else:
1308 d[k] += " " + v
1309 else:
1310 d[k] = v
1311
1312 for k, kinfo in valid_keys.items():
1313 if k not in d:
1314 if "required" in kinfo:
1315 required = kinfo["required"]
1316 else:
1317 required = False
1318
1319 if required:
1320 raise ConfigurationError(
1321 self.filename,
1322 "missing required value for '%s' in test '%s'" %
1323 (k, name))
1324 else:
1325 if "default" in kinfo:
1326 default = kinfo["default"]
1327 else:
1328 default = self._cast_value("", kinfo["type"])
1329 d[k] = default
1330 else:
1331 try:
1332 d[k] = self._cast_value(d[k], kinfo["type"])
1333 except ValueError:
1334 raise ConfigurationError(
1335 self.filename, "bad %s value '%s' for key '%s' in name '%s'" %
1336 (kinfo["type"], d[k], k, name))
1337
1338 return d
1339
1340
1341class Platform:
1342 """Class representing metadata for a particular platform
1343
1344 Maps directly to BOARD when building"""
1345
1346 platform_schema = scl.yaml_load(os.path.join(ZEPHYR_BASE,
1347 "scripts", "sanity_chk", "platform-schema.yaml"))
1348
1349 def __init__(self):
1350 """Constructor.
1351
1352 """
1353
1354 self.name = ""
1355 self.sanitycheck = True
1356 # if no RAM size is specified by the board, take a default of 128K
1357 self.ram = 128
1358
1359 self.ignore_tags = []
Anas Nashife8e367a2020-07-16 16:27:04 -04001360 self.only_tags = []
Anas Nashifce2b4182020-03-24 14:40:28 -04001361 self.default = False
1362 # if no flash size is specified by the board, take a default of 512K
1363 self.flash = 512
1364 self.supported = set()
1365
1366 self.arch = ""
1367 self.type = "na"
1368 self.simulation = "na"
1369 self.supported_toolchains = []
1370 self.env = []
1371 self.env_satisfied = True
1372 self.filter_data = dict()
1373
1374 def load(self, platform_file):
1375 scp = SanityConfigParser(platform_file, self.platform_schema)
1376 scp.load()
1377 data = scp.data
1378
1379 self.name = data['identifier']
1380 self.sanitycheck = data.get("sanitycheck", True)
1381 # if no RAM size is specified by the board, take a default of 128K
1382 self.ram = data.get("ram", 128)
1383 testing = data.get("testing", {})
1384 self.ignore_tags = testing.get("ignore_tags", [])
Anas Nashife8e367a2020-07-16 16:27:04 -04001385 self.only_tags = testing.get("only_tags", [])
Anas Nashifce2b4182020-03-24 14:40:28 -04001386 self.default = testing.get("default", False)
1387 # if no flash size is specified by the board, take a default of 512K
1388 self.flash = data.get("flash", 512)
1389 self.supported = set()
1390 for supp_feature in data.get("supported", []):
1391 for item in supp_feature.split(":"):
1392 self.supported.add(item)
1393
1394 self.arch = data['arch']
1395 self.type = data.get('type', "na")
1396 self.simulation = data.get('simulation', "na")
1397 self.supported_toolchains = data.get("toolchain", [])
1398 self.env = data.get("env", [])
1399 self.env_satisfied = True
1400 for env in self.env:
1401 if not os.environ.get(env, None):
1402 self.env_satisfied = False
1403
1404 def __repr__(self):
1405 return "<%s on %s>" % (self.name, self.arch)
1406
1407
Anas Nashifaff616d2020-04-17 21:24:57 -04001408class DisablePyTestCollectionMixin(object):
1409 __test__ = False
1410
1411
1412class TestCase(DisablePyTestCollectionMixin):
Anas Nashifce2b4182020-03-24 14:40:28 -04001413 """Class representing a test application
1414 """
1415
Anas Nashifaff616d2020-04-17 21:24:57 -04001416 def __init__(self, testcase_root, workdir, name):
Anas Nashifce2b4182020-03-24 14:40:28 -04001417 """TestCase constructor.
1418
1419 This gets called by TestSuite as it finds and reads test yaml files.
1420 Multiple TestCase instances may be generated from a single testcase.yaml,
1421 each one corresponds to an entry within that file.
1422
1423 We need to have a unique name for every single test case. Since
1424 a testcase.yaml can define multiple tests, the canonical name for
1425 the test case is <workdir>/<name>.
1426
1427 @param testcase_root os.path.abspath() of one of the --testcase-root
1428 @param workdir Sub-directory of testcase_root where the
1429 .yaml test configuration file was found
1430 @param name Name of this test case, corresponding to the entry name
1431 in the test case configuration file. For many test cases that just
1432 define one test, can be anything and is usually "test". This is
1433 really only used to distinguish between different cases when
1434 the testcase.yaml defines multiple tests
Anas Nashifce2b4182020-03-24 14:40:28 -04001435 """
1436
Anas Nashifaff616d2020-04-17 21:24:57 -04001437
Anas Nashifce2b4182020-03-24 14:40:28 -04001438 self.source_dir = ""
1439 self.yamlfile = ""
1440 self.cases = []
Anas Nashifaff616d2020-04-17 21:24:57 -04001441 self.name = self.get_unique(testcase_root, workdir, name)
1442 self.id = name
Anas Nashifce2b4182020-03-24 14:40:28 -04001443
1444 self.type = None
Anas Nashifaff616d2020-04-17 21:24:57 -04001445 self.tags = set()
Anas Nashifce2b4182020-03-24 14:40:28 -04001446 self.extra_args = None
1447 self.extra_configs = None
Anas Nashifdca317c2020-08-26 11:28:25 -04001448 self.arch_allow = None
Anas Nashifce2b4182020-03-24 14:40:28 -04001449 self.arch_exclude = None
Anas Nashifaff616d2020-04-17 21:24:57 -04001450 self.skip = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001451 self.platform_exclude = None
Anas Nashifdca317c2020-08-26 11:28:25 -04001452 self.platform_allow = None
Anas Nashifce2b4182020-03-24 14:40:28 -04001453 self.toolchain_exclude = None
Anas Nashifdca317c2020-08-26 11:28:25 -04001454 self.toolchain_allow = None
Anas Nashifce2b4182020-03-24 14:40:28 -04001455 self.tc_filter = None
1456 self.timeout = 60
1457 self.harness = ""
1458 self.harness_config = {}
1459 self.build_only = True
1460 self.build_on_all = False
1461 self.slow = False
Anas Nashifaff616d2020-04-17 21:24:57 -04001462 self.min_ram = -1
Anas Nashifce2b4182020-03-24 14:40:28 -04001463 self.depends_on = None
Anas Nashifaff616d2020-04-17 21:24:57 -04001464 self.min_flash = -1
Anas Nashifce2b4182020-03-24 14:40:28 -04001465 self.extra_sections = None
Anas Nashif1636c312020-05-28 08:02:54 -04001466 self.integration_platforms = []
Anas Nashifce2b4182020-03-24 14:40:28 -04001467
1468 @staticmethod
1469 def get_unique(testcase_root, workdir, name):
1470
1471 canonical_testcase_root = os.path.realpath(testcase_root)
1472 if Path(canonical_zephyr_base) in Path(canonical_testcase_root).parents:
1473 # This is in ZEPHYR_BASE, so include path in name for uniqueness
1474 # FIXME: We should not depend on path of test for unique names.
1475 relative_tc_root = os.path.relpath(canonical_testcase_root,
1476 start=canonical_zephyr_base)
1477 else:
1478 relative_tc_root = ""
1479
1480 # workdir can be "."
1481 unique = os.path.normpath(os.path.join(relative_tc_root, workdir, name))
Anas Nashif7a691252020-05-07 07:47:51 -04001482 check = name.split(".")
1483 if len(check) < 2:
1484 raise SanityCheckException(f"""bad test name '{name}' in {testcase_root}/{workdir}. \
1485Tests should reference the category and subsystem with a dot as a separator.
1486 """
1487 )
Anas Nashifce2b4182020-03-24 14:40:28 -04001488 return unique
1489
1490 @staticmethod
1491 def scan_file(inf_name):
1492 suite_regex = re.compile(
1493 # do not match until end-of-line, otherwise we won't allow
1494 # stc_regex below to catch the ones that are declared in the same
1495 # line--as we only search starting the end of this match
1496 br"^\s*ztest_test_suite\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
1497 re.MULTILINE)
1498 stc_regex = re.compile(
1499 br"^\s*" # empy space at the beginning is ok
1500 # catch the case where it is declared in the same sentence, e.g:
1501 #
1502 # ztest_test_suite(mutex_complex, ztest_user_unit_test(TESTNAME));
1503 br"(?:ztest_test_suite\([a-zA-Z0-9_]+,\s*)?"
1504 # Catch ztest[_user]_unit_test-[_setup_teardown](TESTNAME)
1505 br"ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?"
1506 # Consume the argument that becomes the extra testcse
1507 br"\(\s*"
1508 br"(?P<stc_name>[a-zA-Z0-9_]+)"
1509 # _setup_teardown() variant has two extra arguments that we ignore
1510 br"(?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?"
1511 br"\s*\)",
1512 # We don't check how it finishes; we don't care
1513 re.MULTILINE)
1514 suite_run_regex = re.compile(
1515 br"^\s*ztest_run_test_suite\((?P<suite_name>[a-zA-Z0-9_]+)\)",
1516 re.MULTILINE)
1517 achtung_regex = re.compile(
1518 br"(#ifdef|#endif)",
1519 re.MULTILINE)
1520 warnings = None
1521
1522 with open(inf_name) as inf:
1523 if os.name == 'nt':
1524 mmap_args = {'fileno': inf.fileno(), 'length': 0, 'access': mmap.ACCESS_READ}
1525 else:
1526 mmap_args = {'fileno': inf.fileno(), 'length': 0, 'flags': mmap.MAP_PRIVATE, 'prot': mmap.PROT_READ,
1527 'offset': 0}
1528
1529 with contextlib.closing(mmap.mmap(**mmap_args)) as main_c:
Anas Nashifce2b4182020-03-24 14:40:28 -04001530 suite_regex_match = suite_regex.search(main_c)
1531 if not suite_regex_match:
1532 # can't find ztest_test_suite, maybe a client, because
1533 # it includes ztest.h
1534 return None, None
1535
1536 suite_run_match = suite_run_regex.search(main_c)
1537 if not suite_run_match:
1538 raise ValueError("can't find ztest_run_test_suite")
1539
1540 achtung_matches = re.findall(
1541 achtung_regex,
1542 main_c[suite_regex_match.end():suite_run_match.start()])
1543 if achtung_matches:
1544 warnings = "found invalid %s in ztest_test_suite()" \
Spoorthy Priya Yeraboluad4d4fc2020-06-25 02:57:05 -07001545 % ", ".join(sorted({match.decode() for match in achtung_matches},reverse = True))
Anas Nashifce2b4182020-03-24 14:40:28 -04001546 _matches = re.findall(
1547 stc_regex,
1548 main_c[suite_regex_match.end():suite_run_match.start()])
Anas Nashif44f7ba02020-05-12 12:26:41 -04001549 for match in _matches:
1550 if not match.decode().startswith("test_"):
1551 warnings = "Found a test that does not start with test_"
Maciej Perkowski034d4f22020-08-03 14:11:11 +02001552 matches = [match.decode().replace("test_", "", 1) for match in _matches]
Anas Nashifce2b4182020-03-24 14:40:28 -04001553 return matches, warnings
1554
1555 def scan_path(self, path):
1556 subcases = []
Anas Nashif91fd68d2020-05-08 07:22:58 -04001557 for filename in glob.glob(os.path.join(path, "src", "*.c*")):
Anas Nashifce2b4182020-03-24 14:40:28 -04001558 try:
1559 _subcases, warnings = self.scan_file(filename)
1560 if warnings:
1561 logger.error("%s: %s" % (filename, warnings))
Anas Nashif61c6e2b2020-05-07 07:03:30 -04001562 raise SanityRuntimeError("%s: %s" % (filename, warnings))
Anas Nashifce2b4182020-03-24 14:40:28 -04001563 if _subcases:
1564 subcases += _subcases
1565 except ValueError as e:
1566 logger.error("%s: can't find: %s" % (filename, e))
Anas Nashif61c6e2b2020-05-07 07:03:30 -04001567
Anas Nashifce2b4182020-03-24 14:40:28 -04001568 for filename in glob.glob(os.path.join(path, "*.c")):
1569 try:
1570 _subcases, warnings = self.scan_file(filename)
1571 if warnings:
1572 logger.error("%s: %s" % (filename, warnings))
1573 if _subcases:
1574 subcases += _subcases
1575 except ValueError as e:
1576 logger.error("%s: can't find: %s" % (filename, e))
1577 return subcases
1578
1579 def parse_subcases(self, test_path):
1580 results = self.scan_path(test_path)
1581 for sub in results:
1582 name = "{}.{}".format(self.id, sub)
1583 self.cases.append(name)
1584
1585 if not results:
1586 self.cases.append(self.id)
1587
1588 def __str__(self):
1589 return self.name
1590
1591
Anas Nashifaff616d2020-04-17 21:24:57 -04001592class TestInstance(DisablePyTestCollectionMixin):
Anas Nashifce2b4182020-03-24 14:40:28 -04001593 """Class representing the execution of a particular TestCase on a platform
1594
1595 @param test The TestCase object we want to build/execute
1596 @param platform Platform object that we want to build and run against
1597 @param base_outdir Base directory for all test results. The actual
1598 out directory used is <outdir>/<platform>/<test case name>
1599 """
1600
1601 def __init__(self, testcase, platform, outdir):
1602
1603 self.testcase = testcase
1604 self.platform = platform
1605
1606 self.status = None
1607 self.reason = "Unknown"
1608 self.metrics = dict()
1609 self.handler = None
1610 self.outdir = outdir
1611
1612 self.name = os.path.join(platform.name, testcase.name)
1613 self.build_dir = os.path.join(outdir, platform.name, testcase.name)
1614
Anas Nashifce2b4182020-03-24 14:40:28 -04001615 self.run = False
1616
1617 self.results = {}
1618
1619 def __lt__(self, other):
1620 return self.name < other.name
1621
Anas Nashif4ca0b952020-07-24 09:22:25 -04001622
Anas Nashif405f1b62020-07-27 12:27:13 -04001623 @staticmethod
1624 def testcase_runnable(testcase, fixtures):
Anas Nashif4ca0b952020-07-24 09:22:25 -04001625 can_run = False
1626 # console harness allows us to run the test and capture data.
1627 if testcase.harness in [ 'console', 'ztest']:
Anas Nashif405f1b62020-07-27 12:27:13 -04001628 can_run = True
Anas Nashif4ca0b952020-07-24 09:22:25 -04001629 # if we have a fixture that is also being supplied on the
1630 # command-line, then we need to run the test, not just build it.
1631 fixture = testcase.harness_config.get('fixture')
1632 if fixture:
Anas Nashif405f1b62020-07-27 12:27:13 -04001633 can_run = (fixture in fixtures)
Anas Nashif4ca0b952020-07-24 09:22:25 -04001634
1635 elif testcase.harness:
1636 can_run = False
1637 else:
1638 can_run = True
1639
1640 return can_run
1641
1642
Anas Nashifaff616d2020-04-17 21:24:57 -04001643 # Global testsuite parameters
Anas Nashif405f1b62020-07-27 12:27:13 -04001644 def check_runnable(self, enable_slow=False, filter='buildable', fixtures=[]):
Anas Nashifce2b4182020-03-24 14:40:28 -04001645
1646 # right now we only support building on windows. running is still work
1647 # in progress.
1648 if os.name == 'nt':
Anas Nashif4ca0b952020-07-24 09:22:25 -04001649 return False
Anas Nashifce2b4182020-03-24 14:40:28 -04001650
1651 # we asked for build-only on the command line
Anas Nashif405f1b62020-07-27 12:27:13 -04001652 if self.testcase.build_only:
Anas Nashif4ca0b952020-07-24 09:22:25 -04001653 return False
Anas Nashifce2b4182020-03-24 14:40:28 -04001654
1655 # Do not run slow tests:
1656 skip_slow = self.testcase.slow and not enable_slow
1657 if skip_slow:
Anas Nashif4ca0b952020-07-24 09:22:25 -04001658 return False
Anas Nashifce2b4182020-03-24 14:40:28 -04001659
Anas Nashif4ca0b952020-07-24 09:22:25 -04001660 target_ready = bool(self.testcase.type == "unit" or \
Anas Nashifce2b4182020-03-24 14:40:28 -04001661 self.platform.type == "native" or \
Anas Nashif0f831b12020-09-03 12:33:16 -04001662 self.platform.simulation in ["mdb", "nsim", "renode", "qemu"] or \
Anas Nashif405f1b62020-07-27 12:27:13 -04001663 filter == 'runnable')
Anas Nashifce2b4182020-03-24 14:40:28 -04001664
1665 if self.platform.simulation == "nsim":
1666 if not find_executable("nsimdrv"):
Anas Nashif4ca0b952020-07-24 09:22:25 -04001667 target_ready = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001668
Anas Nashif0f831b12020-09-03 12:33:16 -04001669 if self.platform.simulation == "mdb":
1670 if not find_executable("mdb"):
Anas Nashif405f1b62020-07-27 12:27:13 -04001671 target_ready = False
Anas Nashif0f831b12020-09-03 12:33:16 -04001672
Anas Nashifce2b4182020-03-24 14:40:28 -04001673 if self.platform.simulation == "renode":
1674 if not find_executable("renode"):
Anas Nashif4ca0b952020-07-24 09:22:25 -04001675 target_ready = False
Anas Nashifce2b4182020-03-24 14:40:28 -04001676
Anas Nashif4ca0b952020-07-24 09:22:25 -04001677 testcase_runnable = self.testcase_runnable(self.testcase, fixtures)
Anas Nashifce2b4182020-03-24 14:40:28 -04001678
Anas Nashif4ca0b952020-07-24 09:22:25 -04001679 return testcase_runnable and target_ready
Anas Nashifce2b4182020-03-24 14:40:28 -04001680
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02001681 def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable_coverage=False, coverage_platform=[]):
Anas Nashifce2b4182020-03-24 14:40:28 -04001682 # Create this in a "sanitycheck/" subdirectory otherwise this
1683 # will pass this overlay to kconfig.py *twice* and kconfig.cmake
1684 # will silently give that second time precedence over any
1685 # --extra-args=CONFIG_*
1686 subdir = os.path.join(self.build_dir, "sanitycheck")
Anas Nashifce2b4182020-03-24 14:40:28 -04001687
Kumar Gala51d69312020-09-24 13:28:50 -05001688 content = ""
Anas Nashifce2b4182020-03-24 14:40:28 -04001689
Kumar Gala51d69312020-09-24 13:28:50 -05001690 if self.testcase.extra_configs:
1691 content = "\n".join(self.testcase.extra_configs)
Anas Nashifce2b4182020-03-24 14:40:28 -04001692
Kumar Gala51d69312020-09-24 13:28:50 -05001693 if enable_coverage:
1694 if platform.name in coverage_platform:
1695 content = content + "\nCONFIG_COVERAGE=y"
1696 content = content + "\nCONFIG_COVERAGE_DUMP=y"
Anas Nashifce2b4182020-03-24 14:40:28 -04001697
Kumar Gala51d69312020-09-24 13:28:50 -05001698 if enable_asan:
1699 if platform.type == "native":
1700 content = content + "\nCONFIG_ASAN=y"
Anas Nashifce2b4182020-03-24 14:40:28 -04001701
Kumar Gala51d69312020-09-24 13:28:50 -05001702 if enable_ubsan:
1703 if platform.type == "native":
1704 content = content + "\nCONFIG_UBSAN=y"
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02001705
Kumar Gala51d69312020-09-24 13:28:50 -05001706 if content:
1707 os.makedirs(subdir, exist_ok=True)
1708 file = os.path.join(subdir, "testcase_extra.conf")
1709 with open(file, "w") as f:
1710 f.write(content)
1711
1712 return content
Anas Nashifce2b4182020-03-24 14:40:28 -04001713
1714 def calculate_sizes(self):
1715 """Get the RAM/ROM sizes of a test case.
1716
1717 This can only be run after the instance has been executed by
1718 MakeGenerator, otherwise there won't be any binaries to measure.
1719
1720 @return A SizeCalculator object
1721 """
1722 fns = glob.glob(os.path.join(self.build_dir, "zephyr", "*.elf"))
1723 fns.extend(glob.glob(os.path.join(self.build_dir, "zephyr", "*.exe")))
1724 fns = [x for x in fns if not x.endswith('_prebuilt.elf')]
1725 if len(fns) != 1:
1726 raise BuildError("Missing/multiple output ELF binary")
1727
1728 return SizeCalculator(fns[0], self.testcase.extra_sections)
1729
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02001730 def fill_results_by_status(self):
1731 """Fills results according to self.status
1732
1733 The method is used to propagate the instance level status
1734 to the test cases inside. Useful when the whole instance is skipped
1735 and the info is required also at the test cases level for reporting.
1736 Should be used with caution, e.g. should not be used
1737 to fill all results with passes
1738 """
1739 status_to_verdict = {
1740 'skipped': 'SKIP',
1741 'error': 'BLOCK',
1742 'failure': 'FAILED'
1743 }
1744
1745 for k in self.results:
1746 self.results[k] = status_to_verdict[self.status]
1747
Anas Nashifce2b4182020-03-24 14:40:28 -04001748 def __repr__(self):
1749 return "<TestCase %s on %s>" % (self.testcase.name, self.platform.name)
1750
1751
1752class CMake():
1753 config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
1754 dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
1755
1756 def __init__(self, testcase, platform, source_dir, build_dir):
1757
1758 self.cwd = None
1759 self.capture_output = True
1760
1761 self.defconfig = {}
1762 self.cmake_cache = {}
1763
1764 self.instance = None
1765 self.testcase = testcase
1766 self.platform = platform
1767 self.source_dir = source_dir
1768 self.build_dir = build_dir
1769 self.log = "build.log"
1770 self.generator = None
1771 self.generator_cmd = None
1772
1773 def parse_generated(self):
1774 self.defconfig = {}
1775 return {}
1776
1777 def run_build(self, args=[]):
1778
1779 logger.debug("Building %s for %s" % (self.source_dir, self.platform.name))
1780
1781 cmake_args = []
1782 cmake_args.extend(args)
1783 cmake = shutil.which('cmake')
1784 cmd = [cmake] + cmake_args
1785 kwargs = dict()
1786
1787 if self.capture_output:
1788 kwargs['stdout'] = subprocess.PIPE
1789 # CMake sends the output of message() to stderr unless it's STATUS
1790 kwargs['stderr'] = subprocess.STDOUT
1791
1792 if self.cwd:
1793 kwargs['cwd'] = self.cwd
1794
1795 p = subprocess.Popen(cmd, **kwargs)
1796 out, _ = p.communicate()
1797
1798 results = {}
1799 if p.returncode == 0:
1800 msg = "Finished building %s for %s" % (self.source_dir, self.platform.name)
1801
1802 self.instance.status = "passed"
1803 results = {'msg': msg, "returncode": p.returncode, "instance": self.instance}
1804
1805 if out:
1806 log_msg = out.decode(sys.getdefaultencoding())
1807 with open(os.path.join(self.build_dir, self.log), "a") as log:
1808 log.write(log_msg)
1809
1810 else:
1811 return None
1812 else:
1813 # A real error occurred, raise an exception
1814 if out:
1815 log_msg = out.decode(sys.getdefaultencoding())
1816 with open(os.path.join(self.build_dir, self.log), "a") as log:
1817 log.write(log_msg)
1818
1819 if log_msg:
1820 res = re.findall("region `(FLASH|RAM|SRAM)' overflowed by", log_msg)
1821 if res:
1822 logger.debug("Test skipped due to {} Overflow".format(res[0]))
1823 self.instance.status = "skipped"
1824 self.instance.reason = "{} overflow".format(res[0])
1825 else:
Anas Nashiff04461e2020-06-29 10:07:02 -04001826 self.instance.status = "error"
Anas Nashifce2b4182020-03-24 14:40:28 -04001827 self.instance.reason = "Build failure"
1828
1829 results = {
1830 "returncode": p.returncode,
1831 "instance": self.instance,
1832 }
1833
1834 return results
1835
1836 def run_cmake(self, args=[]):
1837
Anas Nashif50925412020-07-16 17:25:19 -04001838 if self.warnings_as_errors:
1839 ldflags = "-Wl,--fatal-warnings"
1840 cflags = "-Werror"
1841 aflags = "-Wa,--fatal-warnings"
1842 else:
1843 ldflags = cflags = aflags = ""
Anas Nashifce2b4182020-03-24 14:40:28 -04001844
Anas Nashif50925412020-07-16 17:25:19 -04001845 logger.debug("Running cmake on %s for %s" % (self.source_dir, self.platform.name))
Anas Nashifce2b4182020-03-24 14:40:28 -04001846 cmake_args = [
Anas Nashif50925412020-07-16 17:25:19 -04001847 f'-B{self.build_dir}',
1848 f'-S{self.source_dir}',
1849 f'-DEXTRA_CFLAGS="{cflags}"',
1850 f'-DEXTRA_AFLAGS="{aflags}',
1851 f'-DEXTRA_LDFLAGS="{ldflags}"',
1852 f'-G{self.generator}'
Anas Nashifce2b4182020-03-24 14:40:28 -04001853 ]
1854
1855 if self.cmake_only:
1856 cmake_args.append("-DCMAKE_EXPORT_COMPILE_COMMANDS=1")
1857
1858 args = ["-D{}".format(a.replace('"', '')) for a in args]
1859 cmake_args.extend(args)
1860
1861 cmake_opts = ['-DBOARD={}'.format(self.platform.name)]
1862 cmake_args.extend(cmake_opts)
1863
1864
1865 logger.debug("Calling cmake with arguments: {}".format(cmake_args))
1866 cmake = shutil.which('cmake')
1867 cmd = [cmake] + cmake_args
1868 kwargs = dict()
1869
1870 if self.capture_output:
1871 kwargs['stdout'] = subprocess.PIPE
1872 # CMake sends the output of message() to stderr unless it's STATUS
1873 kwargs['stderr'] = subprocess.STDOUT
1874
1875 if self.cwd:
1876 kwargs['cwd'] = self.cwd
1877
1878 p = subprocess.Popen(cmd, **kwargs)
1879 out, _ = p.communicate()
1880
1881 if p.returncode == 0:
1882 filter_results = self.parse_generated()
1883 msg = "Finished building %s for %s" % (self.source_dir, self.platform.name)
1884 logger.debug(msg)
1885 results = {'msg': msg, 'filter': filter_results}
1886
1887 else:
Anas Nashiff04461e2020-06-29 10:07:02 -04001888 self.instance.status = "error"
Anas Nashifce2b4182020-03-24 14:40:28 -04001889 self.instance.reason = "Cmake build failure"
1890 logger.error("Cmake build failure: %s for %s" % (self.source_dir, self.platform.name))
1891 results = {"returncode": p.returncode}
1892
1893 if out:
1894 with open(os.path.join(self.build_dir, self.log), "a") as log:
1895 log_msg = out.decode(sys.getdefaultencoding())
1896 log.write(log_msg)
1897
1898 return results
1899
1900
1901class FilterBuilder(CMake):
1902
1903 def __init__(self, testcase, platform, source_dir, build_dir):
1904 super().__init__(testcase, platform, source_dir, build_dir)
1905
1906 self.log = "config-sanitycheck.log"
1907
1908 def parse_generated(self):
1909
1910 if self.platform.name == "unit_testing":
1911 return {}
1912
1913 cmake_cache_path = os.path.join(self.build_dir, "CMakeCache.txt")
1914 defconfig_path = os.path.join(self.build_dir, "zephyr", ".config")
1915
1916 with open(defconfig_path, "r") as fp:
1917 defconfig = {}
1918 for line in fp.readlines():
1919 m = self.config_re.match(line)
1920 if not m:
1921 if line.strip() and not line.startswith("#"):
1922 sys.stderr.write("Unrecognized line %s\n" % line)
1923 continue
1924 defconfig[m.group(1)] = m.group(2).strip()
1925
1926 self.defconfig = defconfig
1927
1928 cmake_conf = {}
1929 try:
1930 cache = CMakeCache.from_file(cmake_cache_path)
1931 except FileNotFoundError:
1932 cache = {}
1933
1934 for k in iter(cache):
1935 cmake_conf[k.name] = k.value
1936
1937 self.cmake_cache = cmake_conf
1938
1939 filter_data = {
1940 "ARCH": self.platform.arch,
1941 "PLATFORM": self.platform.name
1942 }
1943 filter_data.update(os.environ)
1944 filter_data.update(self.defconfig)
1945 filter_data.update(self.cmake_cache)
1946
Martí Bolívar9c92baa2020-07-08 14:43:07 -07001947 edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle")
Anas Nashifce2b4182020-03-24 14:40:28 -04001948 if self.testcase and self.testcase.tc_filter:
1949 try:
Martí Bolívar9c92baa2020-07-08 14:43:07 -07001950 if os.path.exists(edt_pickle):
1951 with open(edt_pickle, 'rb') as f:
1952 edt = pickle.load(f)
Anas Nashifce2b4182020-03-24 14:40:28 -04001953 else:
1954 edt = None
1955 res = expr_parser.parse(self.testcase.tc_filter, filter_data, edt)
1956
1957 except (ValueError, SyntaxError) as se:
1958 sys.stderr.write(
1959 "Failed processing %s\n" % self.testcase.yamlfile)
1960 raise se
1961
1962 if not res:
1963 return {os.path.join(self.platform.name, self.testcase.name): True}
1964 else:
1965 return {os.path.join(self.platform.name, self.testcase.name): False}
1966 else:
1967 self.platform.filter_data = filter_data
1968 return filter_data
1969
1970
1971class ProjectBuilder(FilterBuilder):
1972
1973 def __init__(self, suite, instance, **kwargs):
1974 super().__init__(instance.testcase, instance.platform, instance.testcase.source_dir, instance.build_dir)
1975
1976 self.log = "build.log"
1977 self.instance = instance
1978 self.suite = suite
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02001979 self.filtered_tests = 0
Anas Nashifce2b4182020-03-24 14:40:28 -04001980
1981 self.lsan = kwargs.get('lsan', False)
1982 self.asan = kwargs.get('asan', False)
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02001983 self.ubsan = kwargs.get('ubsan', False)
Anas Nashifce2b4182020-03-24 14:40:28 -04001984 self.valgrind = kwargs.get('valgrind', False)
1985 self.extra_args = kwargs.get('extra_args', [])
1986 self.device_testing = kwargs.get('device_testing', False)
1987 self.cmake_only = kwargs.get('cmake_only', False)
1988 self.cleanup = kwargs.get('cleanup', False)
1989 self.coverage = kwargs.get('coverage', False)
1990 self.inline_logs = kwargs.get('inline_logs', False)
Anas Nashifce2b4182020-03-24 14:40:28 -04001991 self.generator = kwargs.get('generator', None)
1992 self.generator_cmd = kwargs.get('generator_cmd', None)
Anas Nashiff6462a32020-03-29 19:02:51 -04001993 self.verbose = kwargs.get('verbose', None)
Anas Nashif50925412020-07-16 17:25:19 -04001994 self.warnings_as_errors = kwargs.get('warnings_as_errors', True)
Anas Nashifce2b4182020-03-24 14:40:28 -04001995
1996 @staticmethod
1997 def log_info(filename, inline_logs):
1998 filename = os.path.abspath(os.path.realpath(filename))
1999 if inline_logs:
2000 logger.info("{:-^100}".format(filename))
2001
2002 try:
2003 with open(filename) as fp:
2004 data = fp.read()
2005 except Exception as e:
2006 data = "Unable to read log data (%s)\n" % (str(e))
2007
2008 logger.error(data)
2009
2010 logger.info("{:-^100}".format(filename))
2011 else:
2012 logger.error("see: " + Fore.YELLOW + filename + Fore.RESET)
2013
2014 def log_info_file(self, inline_logs):
2015 build_dir = self.instance.build_dir
2016 h_log = "{}/handler.log".format(build_dir)
2017 b_log = "{}/build.log".format(build_dir)
2018 v_log = "{}/valgrind.log".format(build_dir)
2019 d_log = "{}/device.log".format(build_dir)
2020
2021 if os.path.exists(v_log) and "Valgrind" in self.instance.reason:
2022 self.log_info("{}".format(v_log), inline_logs)
2023 elif os.path.exists(h_log) and os.path.getsize(h_log) > 0:
2024 self.log_info("{}".format(h_log), inline_logs)
2025 elif os.path.exists(d_log) and os.path.getsize(d_log) > 0:
2026 self.log_info("{}".format(d_log), inline_logs)
2027 else:
2028 self.log_info("{}".format(b_log), inline_logs)
2029
2030 def setup_handler(self):
2031
2032 instance = self.instance
2033 args = []
2034
2035 # FIXME: Needs simplification
2036 if instance.platform.simulation == "qemu":
2037 instance.handler = QEMUHandler(instance, "qemu")
2038 args.append("QEMU_PIPE=%s" % instance.handler.get_fifo())
2039 instance.handler.call_make_run = True
2040 elif instance.testcase.type == "unit":
2041 instance.handler = BinaryHandler(instance, "unit")
2042 instance.handler.binary = os.path.join(instance.build_dir, "testbinary")
Anas Nashif051602f2020-04-28 14:27:46 -04002043 if self.coverage:
2044 args.append("COVERAGE=1")
Anas Nashifce2b4182020-03-24 14:40:28 -04002045 elif instance.platform.type == "native":
2046 handler = BinaryHandler(instance, "native")
2047
2048 handler.asan = self.asan
2049 handler.valgrind = self.valgrind
2050 handler.lsan = self.lsan
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002051 handler.ubsan = self.ubsan
Anas Nashifce2b4182020-03-24 14:40:28 -04002052 handler.coverage = self.coverage
2053
2054 handler.binary = os.path.join(instance.build_dir, "zephyr", "zephyr.exe")
2055 instance.handler = handler
2056 elif instance.platform.simulation == "nsim":
2057 if find_executable("nsimdrv"):
2058 instance.handler = BinaryHandler(instance, "nsim")
2059 instance.handler.call_make_run = True
2060 elif instance.platform.simulation == "renode":
2061 if find_executable("renode"):
2062 instance.handler = BinaryHandler(instance, "renode")
2063 instance.handler.pid_fn = os.path.join(instance.build_dir, "renode.pid")
2064 instance.handler.call_make_run = True
2065 elif self.device_testing:
2066 instance.handler = DeviceHandler(instance, "device")
2067
2068 if instance.handler:
2069 instance.handler.args = args
Anas Nashifb3669492020-03-24 22:33:50 -04002070 instance.handler.generator_cmd = self.generator_cmd
2071 instance.handler.generator = self.generator
Anas Nashifce2b4182020-03-24 14:40:28 -04002072
2073 def process(self, message):
2074 op = message.get('op')
2075
2076 if not self.instance.handler:
2077 self.setup_handler()
2078
2079 # The build process, call cmake and build with configured generator
2080 if op == "cmake":
2081 results = self.cmake()
Anas Nashiff04461e2020-06-29 10:07:02 -04002082 if self.instance.status in ["failed", "error"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04002083 pipeline.put({"op": "report", "test": self.instance})
2084 elif self.cmake_only:
Kumar Gala659b24b2020-10-09 09:51:02 -05002085 if self.instance.status is None:
2086 self.instance.status = "passed"
Anas Nashifce2b4182020-03-24 14:40:28 -04002087 pipeline.put({"op": "report", "test": self.instance})
2088 else:
2089 if self.instance.name in results['filter'] and results['filter'][self.instance.name]:
2090 logger.debug("filtering %s" % self.instance.name)
2091 self.instance.status = "skipped"
2092 self.instance.reason = "filter"
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002093 self.suite.build_filtered_tests += 1
Maciej Perkowskib2fa99c2020-05-21 14:45:29 +02002094 for case in self.instance.testcase.cases:
2095 self.instance.results.update({case: 'SKIP'})
Anas Nashifce2b4182020-03-24 14:40:28 -04002096 pipeline.put({"op": "report", "test": self.instance})
2097 else:
2098 pipeline.put({"op": "build", "test": self.instance})
2099
2100 elif op == "build":
2101 logger.debug("build test: %s" % self.instance.name)
2102 results = self.build()
2103
2104 if not results:
Anas Nashiff04461e2020-06-29 10:07:02 -04002105 self.instance.status = "error"
Anas Nashifce2b4182020-03-24 14:40:28 -04002106 self.instance.reason = "Build Failure"
2107 pipeline.put({"op": "report", "test": self.instance})
2108 else:
2109 if results.get('returncode', 1) > 0:
2110 pipeline.put({"op": "report", "test": self.instance})
2111 else:
Anas Nashif405f1b62020-07-27 12:27:13 -04002112 if self.instance.run and self.instance.handler:
Anas Nashifce2b4182020-03-24 14:40:28 -04002113 pipeline.put({"op": "run", "test": self.instance})
2114 else:
2115 pipeline.put({"op": "report", "test": self.instance})
2116 # Run the generated binary using one of the supported handlers
2117 elif op == "run":
2118 logger.debug("run test: %s" % self.instance.name)
2119 self.run()
2120 self.instance.status, _ = self.instance.handler.get_state()
Anas Nashif869ca052020-07-07 14:29:07 -04002121 logger.debug(f"run status: {self.instance.status}")
Anas Nashifce2b4182020-03-24 14:40:28 -04002122 pipeline.put({
2123 "op": "report",
2124 "test": self.instance,
2125 "state": "executed",
2126 "status": self.instance.status,
2127 "reason": self.instance.reason}
2128 )
2129
2130 # Report results and output progress to screen
2131 elif op == "report":
2132 with report_lock:
2133 self.report_out()
2134
2135 if self.cleanup and not self.coverage and self.instance.status == "passed":
2136 pipeline.put({
2137 "op": "cleanup",
2138 "test": self.instance
2139 })
2140
2141 elif op == "cleanup":
Kumar Gala1285c1f2020-09-24 13:21:07 -05002142 if self.device_testing:
2143 self.cleanup_device_testing_artifacts()
2144 else:
2145 self.cleanup_artifacts()
Anas Nashifce2b4182020-03-24 14:40:28 -04002146
Kumar Gala1285c1f2020-09-24 13:21:07 -05002147 def cleanup_artifacts(self, additional_keep=None):
Anas Nashifce2b4182020-03-24 14:40:28 -04002148 logger.debug("Cleaning up {}".format(self.instance.build_dir))
Anas Nashifdca317c2020-08-26 11:28:25 -04002149 allow = [
Anas Nashifce2b4182020-03-24 14:40:28 -04002150 'zephyr/.config',
2151 'handler.log',
2152 'build.log',
2153 'device.log',
Anas Nashif9ace63e2020-04-28 07:14:43 -04002154 'recording.csv',
Anas Nashifce2b4182020-03-24 14:40:28 -04002155 ]
Kumar Gala1285c1f2020-09-24 13:21:07 -05002156
2157 allow += additional_keep
2158
Anas Nashifdca317c2020-08-26 11:28:25 -04002159 allow = [os.path.join(self.instance.build_dir, file) for file in allow]
Anas Nashifce2b4182020-03-24 14:40:28 -04002160
2161 for dirpath, dirnames, filenames in os.walk(self.instance.build_dir, topdown=False):
2162 for name in filenames:
2163 path = os.path.join(dirpath, name)
Anas Nashifdca317c2020-08-26 11:28:25 -04002164 if path not in allow:
Anas Nashifce2b4182020-03-24 14:40:28 -04002165 os.remove(path)
2166 # Remove empty directories and symbolic links to directories
2167 for dir in dirnames:
2168 path = os.path.join(dirpath, dir)
2169 if os.path.islink(path):
2170 os.remove(path)
2171 elif not os.listdir(path):
2172 os.rmdir(path)
2173
Kumar Gala1285c1f2020-09-24 13:21:07 -05002174 def cleanup_device_testing_artifacts(self):
2175 logger.debug("Cleaning up for Device Testing {}".format(self.instance.build_dir))
2176
2177 keep = [
2178 'CMakeCache.txt',
2179 'zephyr/runners.yaml',
2180 'zephyr/zephyr.hex',
2181 'zephyr/zephyr.bin',
2182 'zephyr/zephyr.elf',
2183 ]
2184
2185 self.cleanup_artifacts(keep)
2186
Anas Nashifce2b4182020-03-24 14:40:28 -04002187 def report_out(self):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002188 total_tests_width = len(str(self.suite.total_to_do))
Anas Nashifce2b4182020-03-24 14:40:28 -04002189 self.suite.total_done += 1
2190 instance = self.instance
2191
Anas Nashiff04461e2020-06-29 10:07:02 -04002192 if instance.status in ["error", "failed", "timeout"]:
Anas Nashifdc43c292020-07-09 09:46:45 -04002193 if instance.status == "error":
2194 self.suite.total_errors += 1
Anas Nashifce2b4182020-03-24 14:40:28 -04002195 self.suite.total_failed += 1
Anas Nashiff6462a32020-03-29 19:02:51 -04002196 if self.verbose:
Anas Nashifce2b4182020-03-24 14:40:28 -04002197 status = Fore.RED + "FAILED " + Fore.RESET + instance.reason
2198 else:
2199 print("")
2200 logger.error(
2201 "{:<25} {:<50} {}FAILED{}: {}".format(
2202 instance.platform.name,
2203 instance.testcase.name,
2204 Fore.RED,
2205 Fore.RESET,
2206 instance.reason))
Anas Nashiff6462a32020-03-29 19:02:51 -04002207 if not self.verbose:
Anas Nashifce2b4182020-03-24 14:40:28 -04002208 self.log_info_file(self.inline_logs)
2209 elif instance.status == "skipped":
Anas Nashifce2b4182020-03-24 14:40:28 -04002210 status = Fore.YELLOW + "SKIPPED" + Fore.RESET
Anas Nashif869ca052020-07-07 14:29:07 -04002211 elif instance.status == "passed":
Anas Nashifce2b4182020-03-24 14:40:28 -04002212 status = Fore.GREEN + "PASSED" + Fore.RESET
Anas Nashif869ca052020-07-07 14:29:07 -04002213 else:
2214 logger.debug(f"Unknown status = {instance.status}")
2215 status = Fore.YELLOW + "UNKNOWN" + Fore.RESET
Anas Nashifce2b4182020-03-24 14:40:28 -04002216
Anas Nashiff6462a32020-03-29 19:02:51 -04002217 if self.verbose:
Anas Nashifce2b4182020-03-24 14:40:28 -04002218 if self.cmake_only:
2219 more_info = "cmake"
2220 elif instance.status == "skipped":
2221 more_info = instance.reason
2222 else:
2223 if instance.handler and instance.run:
2224 more_info = instance.handler.type_str
2225 htime = instance.handler.duration
2226 if htime:
2227 more_info += " {:.3f}s".format(htime)
2228 else:
2229 more_info = "build"
2230
2231 logger.info("{:>{}}/{} {:<25} {:<50} {} ({})".format(
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002232 self.suite.total_done, total_tests_width, self.suite.total_to_do, instance.platform.name,
Anas Nashifce2b4182020-03-24 14:40:28 -04002233 instance.testcase.name, status, more_info))
2234
Anas Nashiff04461e2020-06-29 10:07:02 -04002235 if instance.status in ["error", "failed", "timeout"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04002236 self.log_info_file(self.inline_logs)
2237 else:
Maciej Perkowski2ec5ec22020-09-28 12:37:55 +02002238 completed_perc = 0
2239 if self.suite.total_to_do > 0:
2240 completed_perc = int((float(self.suite.total_done) / self.suite.total_to_do) * 100)
2241
Anas Nashifce2b4182020-03-24 14:40:28 -04002242 sys.stdout.write("\rINFO - Total complete: %s%4d/%4d%s %2d%% skipped: %s%4d%s, failed: %s%4d%s" % (
2243 Fore.GREEN,
2244 self.suite.total_done,
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002245 self.suite.total_to_do,
Anas Nashifce2b4182020-03-24 14:40:28 -04002246 Fore.RESET,
Maciej Perkowski2ec5ec22020-09-28 12:37:55 +02002247 completed_perc,
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002248 Fore.YELLOW if self.suite.build_filtered_tests > 0 else Fore.RESET,
2249 self.suite.build_filtered_tests,
Anas Nashifce2b4182020-03-24 14:40:28 -04002250 Fore.RESET,
2251 Fore.RED if self.suite.total_failed > 0 else Fore.RESET,
2252 self.suite.total_failed,
2253 Fore.RESET
2254 )
2255 )
2256 sys.stdout.flush()
2257
2258 def cmake(self):
2259
2260 instance = self.instance
2261 args = self.testcase.extra_args[:]
2262 args += self.extra_args
2263
2264 if instance.handler:
2265 args += instance.handler.args
2266
2267 # merge overlay files into one variable
2268 def extract_overlays(args):
2269 re_overlay = re.compile('OVERLAY_CONFIG=(.*)')
2270 other_args = []
2271 overlays = []
2272 for arg in args:
2273 match = re_overlay.search(arg)
2274 if match:
2275 overlays.append(match.group(1).strip('\'"'))
2276 else:
2277 other_args.append(arg)
2278
2279 args[:] = other_args
2280 return overlays
2281
2282 overlays = extract_overlays(args)
2283
2284 if (self.testcase.extra_configs or self.coverage or
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002285 self.asan or self.ubsan):
Anas Nashifce2b4182020-03-24 14:40:28 -04002286 overlays.append(os.path.join(instance.build_dir,
2287 "sanitycheck", "testcase_extra.conf"))
2288
2289 if overlays:
2290 args.append("OVERLAY_CONFIG=\"%s\"" % (" ".join(overlays)))
2291
2292 results = self.run_cmake(args)
2293 return results
2294
2295 def build(self):
2296 results = self.run_build(['--build', self.build_dir])
2297 return results
2298
2299 def run(self):
2300
2301 instance = self.instance
2302
Anas Nashif405f1b62020-07-27 12:27:13 -04002303 if instance.handler:
2304 if instance.handler.type_str == "device":
2305 instance.handler.suite = self.suite
Anas Nashifce2b4182020-03-24 14:40:28 -04002306
Anas Nashif405f1b62020-07-27 12:27:13 -04002307 instance.handler.handle()
Anas Nashifce2b4182020-03-24 14:40:28 -04002308
2309 sys.stdout.flush()
2310
2311
2312class BoundedExecutor(concurrent.futures.ThreadPoolExecutor):
2313 """BoundedExecutor behaves as a ThreadPoolExecutor which will block on
2314 calls to submit() once the limit given as "bound" work items are queued for
2315 execution.
2316 :param bound: Integer - the maximum number of items in the work queue
2317 :param max_workers: Integer - the size of the thread pool
2318 """
2319
2320 def __init__(self, bound, max_workers, **kwargs):
2321 super().__init__(max_workers)
2322 # self.executor = ThreadPoolExecutor(max_workers=max_workers)
2323 self.semaphore = BoundedSemaphore(bound + max_workers)
2324
2325 def submit(self, fn, *args, **kwargs):
2326 self.semaphore.acquire()
2327 try:
2328 future = super().submit(fn, *args, **kwargs)
2329 except Exception:
2330 self.semaphore.release()
2331 raise
2332 else:
2333 future.add_done_callback(lambda x: self.semaphore.release())
2334 return future
2335
2336
Anas Nashifaff616d2020-04-17 21:24:57 -04002337class TestSuite(DisablePyTestCollectionMixin):
Anas Nashifce2b4182020-03-24 14:40:28 -04002338 config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
2339 dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
2340
2341 tc_schema = scl.yaml_load(
2342 os.path.join(ZEPHYR_BASE,
2343 "scripts", "sanity_chk", "testcase-schema.yaml"))
2344
2345 testcase_valid_keys = {"tags": {"type": "set", "required": False},
2346 "type": {"type": "str", "default": "integration"},
2347 "extra_args": {"type": "list"},
2348 "extra_configs": {"type": "list"},
2349 "build_only": {"type": "bool", "default": False},
2350 "build_on_all": {"type": "bool", "default": False},
2351 "skip": {"type": "bool", "default": False},
2352 "slow": {"type": "bool", "default": False},
2353 "timeout": {"type": "int", "default": 60},
2354 "min_ram": {"type": "int", "default": 8},
2355 "depends_on": {"type": "set"},
2356 "min_flash": {"type": "int", "default": 32},
Anas Nashifdca317c2020-08-26 11:28:25 -04002357 "arch_allow": {"type": "set"},
Anas Nashifce2b4182020-03-24 14:40:28 -04002358 "arch_exclude": {"type": "set"},
2359 "extra_sections": {"type": "list", "default": []},
Anas Nashif1636c312020-05-28 08:02:54 -04002360 "integration_platforms": {"type": "list", "default": []},
Anas Nashifce2b4182020-03-24 14:40:28 -04002361 "platform_exclude": {"type": "set"},
Anas Nashifdca317c2020-08-26 11:28:25 -04002362 "platform_allow": {"type": "set"},
Anas Nashifce2b4182020-03-24 14:40:28 -04002363 "toolchain_exclude": {"type": "set"},
Anas Nashifdca317c2020-08-26 11:28:25 -04002364 "toolchain_allow": {"type": "set"},
Anas Nashifce2b4182020-03-24 14:40:28 -04002365 "filter": {"type": "str"},
2366 "harness": {"type": "str"},
2367 "harness_config": {"type": "map", "default": {}}
2368 }
2369
2370 RELEASE_DATA = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk",
2371 "sanity_last_release.csv")
2372
Aastha Grovera0ae5342020-05-13 13:34:00 -07002373 SAMPLE_FILENAME = 'sample.yaml'
2374 TESTCASE_FILENAME = 'testcase.yaml'
2375
Anas Nashifaff616d2020-04-17 21:24:57 -04002376 def __init__(self, board_root_list=[], testcase_roots=[], outdir=None):
Anas Nashifce2b4182020-03-24 14:40:28 -04002377
2378 self.roots = testcase_roots
2379 if not isinstance(board_root_list, list):
2380 self.board_roots = [board_root_list]
2381 else:
2382 self.board_roots = board_root_list
2383
2384 # Testsuite Options
2385 self.coverage_platform = []
2386 self.build_only = False
2387 self.cmake_only = False
2388 self.cleanup = False
2389 self.enable_slow = False
2390 self.device_testing = False
Anas Nashifce8c12e2020-05-21 09:11:40 -04002391 self.fixtures = []
Anas Nashifce2b4182020-03-24 14:40:28 -04002392 self.enable_coverage = False
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002393 self.enable_ubsan = False
Anas Nashifce2b4182020-03-24 14:40:28 -04002394 self.enable_lsan = False
2395 self.enable_asan = False
2396 self.enable_valgrind = False
2397 self.extra_args = []
2398 self.inline_logs = False
2399 self.enable_sizes_report = False
2400 self.west_flash = None
2401 self.west_runner = None
2402 self.generator = None
2403 self.generator_cmd = None
Anas Nashif50925412020-07-16 17:25:19 -04002404 self.warnings_as_errors = True
Anas Nashifce2b4182020-03-24 14:40:28 -04002405
2406 # Keep track of which test cases we've filtered out and why
2407 self.testcases = {}
2408 self.platforms = []
2409 self.selected_platforms = []
2410 self.default_platforms = []
2411 self.outdir = os.path.abspath(outdir)
Anas Nashifaff616d2020-04-17 21:24:57 -04002412 self.discards = {}
Anas Nashifce2b4182020-03-24 14:40:28 -04002413 self.load_errors = 0
2414 self.instances = dict()
2415
2416 self.total_tests = 0 # number of test instances
2417 self.total_cases = 0 # number of test cases
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002418 self.total_skipped_cases = 0 # number of skipped test cases
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002419 self.total_to_do = 0 # number of test instances to be run
Anas Nashifce2b4182020-03-24 14:40:28 -04002420 self.total_done = 0 # tests completed
2421 self.total_failed = 0
2422 self.total_skipped = 0
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002423 self.build_filtered_tests = 0
Anas Nashifdc43c292020-07-09 09:46:45 -04002424 self.total_passed = 0
2425 self.total_errors = 0
Anas Nashifce2b4182020-03-24 14:40:28 -04002426
2427 self.total_platforms = 0
2428 self.start_time = 0
2429 self.duration = 0
2430 self.warnings = 0
2431 self.cv = threading.Condition()
2432
2433 # hardcoded for now
2434 self.connected_hardware = []
2435
Anas Nashif1636c312020-05-28 08:02:54 -04002436 # run integration tests only
2437 self.integration = False
2438
Anas Nashifbb280352020-05-07 12:02:48 -04002439 def get_platform_instances(self, platform):
2440 filtered_dict = {k:v for k,v in self.instances.items() if k.startswith(platform + "/")}
2441 return filtered_dict
2442
Anas Nashifce2b4182020-03-24 14:40:28 -04002443 def config(self):
2444 logger.info("coverage platform: {}".format(self.coverage_platform))
2445
2446 # Debug Functions
2447 @staticmethod
2448 def info(what):
2449 sys.stdout.write(what + "\n")
2450 sys.stdout.flush()
2451
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002452 def update_counting(self):
Anas Nashifce2b4182020-03-24 14:40:28 -04002453 self.total_tests = len(self.instances)
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002454 self.total_cases = 0
Maciej Perkowskic67a0cd2020-08-13 15:20:13 +02002455 self.total_skipped = 0
2456 self.total_skipped_cases = 0
2457 self.total_passed = 0
2458 for instance in self.instances.values():
2459 self.total_cases += len(instance.testcase.cases)
2460 if instance.status == 'skipped':
2461 self.total_skipped += 1
2462 self.total_skipped_cases += len(instance.testcase.cases)
2463 elif instance.status == "passed":
2464 self.total_passed += 1
2465 for res in instance.results.values():
2466 if res == 'SKIP':
2467 self.total_skipped_cases += 1
2468 self.total_to_do = self.total_tests - self.total_skipped
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002469
Anas Nashifce2b4182020-03-24 14:40:28 -04002470
2471 def compare_metrics(self, filename):
2472 # name, datatype, lower results better
2473 interesting_metrics = [("ram_size", int, True),
2474 ("rom_size", int, True)]
2475
2476 if not os.path.exists(filename):
Anas Nashifad44bed2020-08-24 18:59:01 -04002477 logger.error("Cannot compare metrics, %s not found" % filename)
Anas Nashifce2b4182020-03-24 14:40:28 -04002478 return []
2479
2480 results = []
2481 saved_metrics = {}
2482 with open(filename) as fp:
2483 cr = csv.DictReader(fp)
2484 for row in cr:
2485 d = {}
2486 for m, _, _ in interesting_metrics:
2487 d[m] = row[m]
2488 saved_metrics[(row["test"], row["platform"])] = d
2489
2490 for instance in self.instances.values():
2491 mkey = (instance.testcase.name, instance.platform.name)
2492 if mkey not in saved_metrics:
2493 continue
2494 sm = saved_metrics[mkey]
2495 for metric, mtype, lower_better in interesting_metrics:
2496 if metric not in instance.metrics:
2497 continue
2498 if sm[metric] == "":
2499 continue
2500 delta = instance.metrics.get(metric, 0) - mtype(sm[metric])
2501 if delta == 0:
2502 continue
2503 results.append((instance, metric, instance.metrics.get(metric, 0), delta,
2504 lower_better))
2505 return results
2506
Anas Nashifad44bed2020-08-24 18:59:01 -04002507 def footprint_reports(self, report, show_footprint, all_deltas,
2508 footprint_threshold, last_metrics):
Anas Nashifce2b4182020-03-24 14:40:28 -04002509 if not report:
2510 return
2511
Anas Nashifad44bed2020-08-24 18:59:01 -04002512 logger.debug("running footprint_reports")
Anas Nashifce2b4182020-03-24 14:40:28 -04002513 deltas = self.compare_metrics(report)
2514 warnings = 0
2515 if deltas and show_footprint:
2516 for i, metric, value, delta, lower_better in deltas:
2517 if not all_deltas and ((delta < 0 and lower_better) or
2518 (delta > 0 and not lower_better)):
2519 continue
2520
Anas Nashifad44bed2020-08-24 18:59:01 -04002521 percentage = 0
2522 if value > delta:
2523 percentage = (float(delta) / float(value - delta))
2524
2525 if not all_deltas and (percentage < (footprint_threshold / 100.0)):
Anas Nashifce2b4182020-03-24 14:40:28 -04002526 continue
2527
2528 logger.info("{:<25} {:<60} {}{}{}: {} {:<+4}, is now {:6} {:+.2%}".format(
2529 i.platform.name, i.testcase.name, Fore.YELLOW,
2530 "INFO" if all_deltas else "WARNING", Fore.RESET,
2531 metric, delta, value, percentage))
2532 warnings += 1
2533
2534 if warnings:
2535 logger.warning("Deltas based on metrics from last %s" %
2536 ("release" if not last_metrics else "run"))
2537
2538 def summary(self, unrecognized_sections):
2539 failed = 0
Anas Nashif4258d8d2020-05-08 08:40:27 -04002540 run = 0
Anas Nashifce2b4182020-03-24 14:40:28 -04002541 for instance in self.instances.values():
2542 if instance.status == "failed":
2543 failed += 1
2544 elif instance.metrics.get("unrecognized") and not unrecognized_sections:
2545 logger.error("%sFAILED%s: %s has unrecognized binary sections: %s" %
2546 (Fore.RED, Fore.RESET, instance.name,
2547 str(instance.metrics.get("unrecognized", []))))
2548 failed += 1
2549
Anas Nashifad44bed2020-08-24 18:59:01 -04002550 if instance.metrics.get('handler_time', None):
Anas Nashif4258d8d2020-05-08 08:40:27 -04002551 run += 1
2552
Anas Nashifce2b4182020-03-24 14:40:28 -04002553 if self.total_tests and self.total_tests != self.total_skipped:
Anas Nashifdc43c292020-07-09 09:46:45 -04002554 pass_rate = (float(self.total_passed) / float(
Anas Nashifce2b4182020-03-24 14:40:28 -04002555 self.total_tests - self.total_skipped))
2556 else:
2557 pass_rate = 0
2558
2559 logger.info(
2560 "{}{} of {}{} tests passed ({:.2%}), {}{}{} failed, {} skipped with {}{}{} warnings in {:.2f} seconds".format(
2561 Fore.RED if failed else Fore.GREEN,
Anas Nashifdc43c292020-07-09 09:46:45 -04002562 self.total_passed,
Anas Nashifce2b4182020-03-24 14:40:28 -04002563 self.total_tests - self.total_skipped,
2564 Fore.RESET,
2565 pass_rate,
2566 Fore.RED if self.total_failed else Fore.RESET,
2567 self.total_failed,
2568 Fore.RESET,
2569 self.total_skipped,
2570 Fore.YELLOW if self.warnings else Fore.RESET,
2571 self.warnings,
2572 Fore.RESET,
2573 self.duration))
2574
2575 self.total_platforms = len(self.platforms)
2576 if self.platforms:
2577 logger.info("In total {} test cases were executed on {} out of total {} platforms ({:02.2f}%)".format(
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002578 self.total_cases - self.total_skipped_cases,
Anas Nashifce2b4182020-03-24 14:40:28 -04002579 len(self.selected_platforms),
2580 self.total_platforms,
2581 (100 * len(self.selected_platforms) / len(self.platforms))
2582 ))
2583
Anas Nashif4258d8d2020-05-08 08:40:27 -04002584 logger.info(f"{Fore.GREEN}{run}{Fore.RESET} tests executed on platforms, \
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002585{Fore.RED}{self.total_tests - run - self.total_skipped}{Fore.RESET} tests were only built.")
Anas Nashif4258d8d2020-05-08 08:40:27 -04002586
Anas Nashif6915adf2020-04-22 09:39:42 -04002587 def save_reports(self, name, suffix, report_dir, no_update, release, only_failed):
Anas Nashifce2b4182020-03-24 14:40:28 -04002588 if not self.instances:
2589 return
2590
2591 if name:
2592 report_name = name
2593 else:
2594 report_name = "sanitycheck"
2595
2596 if report_dir:
2597 os.makedirs(report_dir, exist_ok=True)
2598 filename = os.path.join(report_dir, report_name)
2599 outdir = report_dir
2600 else:
2601 filename = os.path.join(self.outdir, report_name)
2602 outdir = self.outdir
2603
Anas Nashif6915adf2020-04-22 09:39:42 -04002604 if suffix:
2605 filename = "{}_{}".format(filename, suffix)
2606
Anas Nashifce2b4182020-03-24 14:40:28 -04002607 if not no_update:
Anas Nashif90415502020-04-11 22:15:04 -04002608 self.xunit_report(filename + ".xml", full_report=False, append=only_failed)
2609 self.xunit_report(filename + "_report.xml", full_report=True, append=only_failed)
Anas Nashifce2b4182020-03-24 14:40:28 -04002610 self.csv_report(filename + ".csv")
Anas Nashif90415502020-04-11 22:15:04 -04002611
Anas Nashif6915adf2020-04-22 09:39:42 -04002612 self.target_report(outdir, suffix, append=only_failed)
Anas Nashifce2b4182020-03-24 14:40:28 -04002613 if self.discards:
2614 self.discard_report(filename + "_discard.csv")
2615
2616 if release:
2617 self.csv_report(self.RELEASE_DATA)
2618
2619 def add_configurations(self):
2620
2621 for board_root in self.board_roots:
2622 board_root = os.path.abspath(board_root)
2623
2624 logger.debug("Reading platform configuration files under %s..." %
2625 board_root)
2626
2627 for file in glob.glob(os.path.join(board_root, "*", "*", "*.yaml")):
2628 logger.debug("Found platform configuration " + file)
2629 try:
2630 platform = Platform()
2631 platform.load(file)
Anas Nashif61c4a512020-08-13 07:46:45 -04002632 if platform.name in [p.name for p in self.platforms]:
2633 logger.error(f"Duplicate platform {platform.name} in {file}")
2634 raise Exception(f"Duplicate platform identifier {platform.name} found")
Anas Nashifce2b4182020-03-24 14:40:28 -04002635 if platform.sanitycheck:
2636 self.platforms.append(platform)
2637 if platform.default:
2638 self.default_platforms.append(platform.name)
2639
2640 except RuntimeError as e:
2641 logger.error("E: %s: can't load: %s" % (file, e))
2642 self.load_errors += 1
2643
2644 def get_all_tests(self):
2645 tests = []
2646 for _, tc in self.testcases.items():
2647 for case in tc.cases:
2648 tests.append(case)
2649
2650 return tests
2651
2652 @staticmethod
2653 def get_toolchain():
2654 toolchain = os.environ.get("ZEPHYR_TOOLCHAIN_VARIANT", None) or \
2655 os.environ.get("ZEPHYR_GCC_VARIANT", None)
2656
2657 if toolchain == "gccarmemb":
2658 # Remove this translation when gccarmemb is no longer supported.
2659 toolchain = "gnuarmemb"
2660
2661 try:
2662 if not toolchain:
2663 raise SanityRuntimeError("E: Variable ZEPHYR_TOOLCHAIN_VARIANT is not defined")
2664 except Exception as e:
2665 print(str(e))
2666 sys.exit(2)
2667
2668 return toolchain
2669
2670 def add_testcases(self, testcase_filter=[]):
2671 for root in self.roots:
2672 root = os.path.abspath(root)
2673
2674 logger.debug("Reading test case configuration files under %s..." % root)
2675
2676 for dirpath, dirnames, filenames in os.walk(root, topdown=True):
2677 logger.debug("scanning %s" % dirpath)
Aastha Grovera0ae5342020-05-13 13:34:00 -07002678 if self.SAMPLE_FILENAME in filenames:
2679 filename = self.SAMPLE_FILENAME
2680 elif self.TESTCASE_FILENAME in filenames:
2681 filename = self.TESTCASE_FILENAME
Anas Nashifce2b4182020-03-24 14:40:28 -04002682 else:
2683 continue
2684
2685 logger.debug("Found possible test case in " + dirpath)
2686
2687 dirnames[:] = []
2688 tc_path = os.path.join(dirpath, filename)
2689
2690 try:
2691 parsed_data = SanityConfigParser(tc_path, self.tc_schema)
2692 parsed_data.load()
2693
2694 tc_path = os.path.dirname(tc_path)
2695 workdir = os.path.relpath(tc_path, root)
2696
2697 for name in parsed_data.tests.keys():
Anas Nashifaff616d2020-04-17 21:24:57 -04002698 tc = TestCase(root, workdir, name)
Anas Nashifce2b4182020-03-24 14:40:28 -04002699
2700 tc_dict = parsed_data.get_test(name, self.testcase_valid_keys)
2701
2702 tc.source_dir = tc_path
2703 tc.yamlfile = tc_path
2704
Anas Nashifce2b4182020-03-24 14:40:28 -04002705 tc.type = tc_dict["type"]
2706 tc.tags = tc_dict["tags"]
2707 tc.extra_args = tc_dict["extra_args"]
2708 tc.extra_configs = tc_dict["extra_configs"]
Anas Nashifdca317c2020-08-26 11:28:25 -04002709 tc.arch_allow = tc_dict["arch_allow"]
Anas Nashifce2b4182020-03-24 14:40:28 -04002710 tc.arch_exclude = tc_dict["arch_exclude"]
2711 tc.skip = tc_dict["skip"]
2712 tc.platform_exclude = tc_dict["platform_exclude"]
Anas Nashifdca317c2020-08-26 11:28:25 -04002713 tc.platform_allow = tc_dict["platform_allow"]
Anas Nashifce2b4182020-03-24 14:40:28 -04002714 tc.toolchain_exclude = tc_dict["toolchain_exclude"]
Anas Nashifdca317c2020-08-26 11:28:25 -04002715 tc.toolchain_allow = tc_dict["toolchain_allow"]
Anas Nashifce2b4182020-03-24 14:40:28 -04002716 tc.tc_filter = tc_dict["filter"]
2717 tc.timeout = tc_dict["timeout"]
2718 tc.harness = tc_dict["harness"]
2719 tc.harness_config = tc_dict["harness_config"]
Anas Nashif43275c82020-05-04 18:22:16 -04002720 if tc.harness == 'console' and not tc.harness_config:
2721 raise Exception('Harness config error: console harness defined without a configuration.')
Anas Nashifce2b4182020-03-24 14:40:28 -04002722 tc.build_only = tc_dict["build_only"]
2723 tc.build_on_all = tc_dict["build_on_all"]
2724 tc.slow = tc_dict["slow"]
2725 tc.min_ram = tc_dict["min_ram"]
2726 tc.depends_on = tc_dict["depends_on"]
2727 tc.min_flash = tc_dict["min_flash"]
2728 tc.extra_sections = tc_dict["extra_sections"]
Anas Nashif1636c312020-05-28 08:02:54 -04002729 tc.integration_platforms = tc_dict["integration_platforms"]
Anas Nashifce2b4182020-03-24 14:40:28 -04002730
2731 tc.parse_subcases(tc_path)
2732
2733 if testcase_filter:
2734 if tc.name and tc.name in testcase_filter:
2735 self.testcases[tc.name] = tc
2736 else:
2737 self.testcases[tc.name] = tc
2738
2739 except Exception as e:
2740 logger.error("%s: can't load (skipping): %s" % (tc_path, e))
2741 self.load_errors += 1
2742
2743
2744 def get_platform(self, name):
2745 selected_platform = None
2746 for platform in self.platforms:
2747 if platform.name == name:
2748 selected_platform = platform
2749 break
2750 return selected_platform
2751
2752 def load_from_file(self, file, filter_status=[]):
2753 try:
2754 with open(file, "r") as fp:
2755 cr = csv.DictReader(fp)
2756 instance_list = []
2757 for row in cr:
2758 if row["status"] in filter_status:
2759 continue
2760 test = row["test"]
2761
2762 platform = self.get_platform(row["platform"])
2763 instance = TestInstance(self.testcases[test], platform, self.outdir)
Anas Nashif405f1b62020-07-27 12:27:13 -04002764 if self.device_testing:
2765 tfilter = 'runnable'
2766 else:
2767 tfilter = 'buildable'
2768 instance.run = instance.check_runnable(
Anas Nashifce2b4182020-03-24 14:40:28 -04002769 self.enable_slow,
Anas Nashif405f1b62020-07-27 12:27:13 -04002770 tfilter,
Anas Nashifce8c12e2020-05-21 09:11:40 -04002771 self.fixtures
Anas Nashifce2b4182020-03-24 14:40:28 -04002772 )
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002773 instance.create_overlay(platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform)
Anas Nashifce2b4182020-03-24 14:40:28 -04002774 instance_list.append(instance)
2775 self.add_instances(instance_list)
2776
2777 except KeyError as e:
2778 logger.error("Key error while parsing tests file.({})".format(str(e)))
2779 sys.exit(2)
2780
2781 except FileNotFoundError as e:
2782 logger.error("Couldn't find input file with list of tests. ({})".format(e))
2783 sys.exit(2)
2784
2785 def apply_filters(self, **kwargs):
2786
2787 toolchain = self.get_toolchain()
2788
2789 discards = {}
2790 platform_filter = kwargs.get('platform')
Anas Nashifaff616d2020-04-17 21:24:57 -04002791 exclude_platform = kwargs.get('exclude_platform', [])
2792 testcase_filter = kwargs.get('run_individual_tests', [])
Anas Nashifce2b4182020-03-24 14:40:28 -04002793 arch_filter = kwargs.get('arch')
2794 tag_filter = kwargs.get('tag')
2795 exclude_tag = kwargs.get('exclude_tag')
2796 all_filter = kwargs.get('all')
Anas Nashif405f1b62020-07-27 12:27:13 -04002797 runnable = kwargs.get('runnable')
Anas Nashifce2b4182020-03-24 14:40:28 -04002798 force_toolchain = kwargs.get('force_toolchain')
Anas Nashif1a5defa2020-05-01 14:57:00 -04002799 force_platform = kwargs.get('force_platform')
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04002800 emu_filter = kwargs.get('emulation_only')
Anas Nashifce2b4182020-03-24 14:40:28 -04002801
2802 logger.debug("platform filter: " + str(platform_filter))
2803 logger.debug(" arch_filter: " + str(arch_filter))
2804 logger.debug(" tag_filter: " + str(tag_filter))
2805 logger.debug(" exclude_tag: " + str(exclude_tag))
2806
2807 default_platforms = False
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04002808 emulation_platforms = False
Anas Nashifce2b4182020-03-24 14:40:28 -04002809
2810 if platform_filter:
2811 platforms = list(filter(lambda p: p.name in platform_filter, self.platforms))
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04002812 elif emu_filter:
2813 platforms = list(filter(lambda p: p.simulation != 'na', self.platforms))
Anas Nashifce2b4182020-03-24 14:40:28 -04002814 else:
2815 platforms = self.platforms
2816
2817 if all_filter:
2818 logger.info("Selecting all possible platforms per test case")
2819 # When --all used, any --platform arguments ignored
2820 platform_filter = []
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04002821 elif not platform_filter and not emu_filter:
Anas Nashifce2b4182020-03-24 14:40:28 -04002822 logger.info("Selecting default platforms per test case")
2823 default_platforms = True
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04002824 elif emu_filter:
2825 logger.info("Selecting emulation platforms per test case")
2826 emulation_platforms = True
Anas Nashifce2b4182020-03-24 14:40:28 -04002827
2828 logger.info("Building initial testcase list...")
2829
2830 for tc_name, tc in self.testcases.items():
2831 # list of instances per testcase, aka configurations.
2832 instance_list = []
2833 for plat in platforms:
2834 instance = TestInstance(tc, plat, self.outdir)
Anas Nashif405f1b62020-07-27 12:27:13 -04002835 if runnable:
2836 tfilter = 'runnable'
2837 else:
2838 tfilter = 'buildable'
2839
2840 instance.run = instance.check_runnable(
Anas Nashifce2b4182020-03-24 14:40:28 -04002841 self.enable_slow,
Anas Nashif405f1b62020-07-27 12:27:13 -04002842 tfilter,
Anas Nashifce8c12e2020-05-21 09:11:40 -04002843 self.fixtures
Anas Nashifce2b4182020-03-24 14:40:28 -04002844 )
Anas Nashiff04461e2020-06-29 10:07:02 -04002845 for t in tc.cases:
2846 instance.results[t] = None
Anas Nashif3b86f132020-05-21 10:35:33 -04002847
Anas Nashif405f1b62020-07-27 12:27:13 -04002848 if runnable and self.connected_hardware:
Anas Nashif3b86f132020-05-21 10:35:33 -04002849 for h in self.connected_hardware:
2850 if h['platform'] == plat.name:
2851 if tc.harness_config.get('fixture') in h.get('fixtures', []):
Anas Nashif3b86f132020-05-21 10:35:33 -04002852 instance.run = True
2853
Anas Nashif1a5defa2020-05-01 14:57:00 -04002854 if not force_platform and plat.name in exclude_platform:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002855 discards[instance] = discards.get(instance, "Platform is excluded on command line.")
Anas Nashifce2b4182020-03-24 14:40:28 -04002856
2857 if (plat.arch == "unit") != (tc.type == "unit"):
2858 # Discard silently
2859 continue
2860
Anas Nashif405f1b62020-07-27 12:27:13 -04002861 if runnable and not instance.run:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002862 discards[instance] = discards.get(instance, "Not runnable on device")
Anas Nashifce2b4182020-03-24 14:40:28 -04002863
Anas Nashif1636c312020-05-28 08:02:54 -04002864 if self.integration and tc.integration_platforms and plat.name not in tc.integration_platforms:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002865 discards[instance] = discards.get(instance, "Not part of integration platforms")
Anas Nashif1636c312020-05-28 08:02:54 -04002866
Anas Nashifce2b4182020-03-24 14:40:28 -04002867 if tc.skip:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002868 discards[instance] = discards.get(instance, "Skip filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04002869
2870 if tc.build_on_all and not platform_filter:
2871 platform_filter = []
2872
2873 if tag_filter and not tc.tags.intersection(tag_filter):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002874 discards[instance] = discards.get(instance, "Command line testcase tag filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04002875
2876 if exclude_tag and tc.tags.intersection(exclude_tag):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002877 discards[instance] = discards.get(instance, "Command line testcase exclude filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04002878
2879 if testcase_filter and tc_name not in testcase_filter:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002880 discards[instance] = discards.get(instance, "Testcase name filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04002881
2882 if arch_filter and plat.arch not in arch_filter:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002883 discards[instance] = discards.get(instance, "Command line testcase arch filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04002884
Anas Nashif1a5defa2020-05-01 14:57:00 -04002885 if not force_platform:
Anas Nashifce2b4182020-03-24 14:40:28 -04002886
Anas Nashifdca317c2020-08-26 11:28:25 -04002887 if tc.arch_allow and plat.arch not in tc.arch_allow:
2888 discards[instance] = discards.get(instance, "Not in test case arch allow list")
Anas Nashifce2b4182020-03-24 14:40:28 -04002889
Anas Nashif1a5defa2020-05-01 14:57:00 -04002890 if tc.arch_exclude and plat.arch in tc.arch_exclude:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002891 discards[instance] = discards.get(instance, "In test case arch exclude")
Anas Nashif1a5defa2020-05-01 14:57:00 -04002892
2893 if tc.platform_exclude and plat.name in tc.platform_exclude:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002894 discards[instance] = discards.get(instance, "In test case platform exclude")
Anas Nashifce2b4182020-03-24 14:40:28 -04002895
2896 if tc.toolchain_exclude and toolchain in tc.toolchain_exclude:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002897 discards[instance] = discards.get(instance, "In test case toolchain exclude")
Anas Nashifce2b4182020-03-24 14:40:28 -04002898
2899 if platform_filter and plat.name not in platform_filter:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002900 discards[instance] = discards.get(instance, "Command line platform filter")
Anas Nashifce2b4182020-03-24 14:40:28 -04002901
Anas Nashifdca317c2020-08-26 11:28:25 -04002902 if tc.platform_allow and plat.name not in tc.platform_allow:
2903 discards[instance] = discards.get(instance, "Not in testcase platform allow list")
Anas Nashifce2b4182020-03-24 14:40:28 -04002904
Anas Nashifdca317c2020-08-26 11:28:25 -04002905 if tc.toolchain_allow and toolchain not in tc.toolchain_allow:
2906 discards[instance] = discards.get(instance, "Not in testcase toolchain allow list")
Anas Nashifce2b4182020-03-24 14:40:28 -04002907
2908 if not plat.env_satisfied:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002909 discards[instance] = discards.get(instance, "Environment ({}) not satisfied".format(", ".join(plat.env)))
Anas Nashifce2b4182020-03-24 14:40:28 -04002910
2911 if not force_toolchain \
2912 and toolchain and (toolchain not in plat.supported_toolchains) \
2913 and tc.type != 'unit':
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002914 discards[instance] = discards.get(instance, "Not supported by the toolchain")
Anas Nashifce2b4182020-03-24 14:40:28 -04002915
2916 if plat.ram < tc.min_ram:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002917 discards[instance] = discards.get(instance, "Not enough RAM")
Anas Nashifce2b4182020-03-24 14:40:28 -04002918
2919 if tc.depends_on:
2920 dep_intersection = tc.depends_on.intersection(set(plat.supported))
2921 if dep_intersection != set(tc.depends_on):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002922 discards[instance] = discards.get(instance, "No hardware support")
Anas Nashifce2b4182020-03-24 14:40:28 -04002923
2924 if plat.flash < tc.min_flash:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002925 discards[instance] = discards.get(instance, "Not enough FLASH")
Anas Nashifce2b4182020-03-24 14:40:28 -04002926
2927 if set(plat.ignore_tags) & tc.tags:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002928 discards[instance] = discards.get(instance, "Excluded tags per platform (exclude_tags)")
Anas Nashife8e367a2020-07-16 16:27:04 -04002929
Anas Nashif555fc6d2020-07-30 07:23:54 -04002930 if plat.only_tags and not set(plat.only_tags) & tc.tags:
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002931 discards[instance] = discards.get(instance, "Excluded tags per platform (only_tags)")
Anas Nashifce2b4182020-03-24 14:40:28 -04002932
2933 # if nothing stopped us until now, it means this configuration
2934 # needs to be added.
2935 instance_list.append(instance)
2936
2937 # no configurations, so jump to next testcase
2938 if not instance_list:
2939 continue
2940
2941 # if sanitycheck was launched with no platform options at all, we
2942 # take all default platforms
2943 if default_platforms and not tc.build_on_all:
Anas Nashifdca317c2020-08-26 11:28:25 -04002944 if tc.platform_allow:
Anas Nashifce2b4182020-03-24 14:40:28 -04002945 a = set(self.default_platforms)
Anas Nashifdca317c2020-08-26 11:28:25 -04002946 b = set(tc.platform_allow)
Anas Nashifce2b4182020-03-24 14:40:28 -04002947 c = a.intersection(b)
2948 if c:
2949 aa = list(filter(lambda tc: tc.platform.name in c, instance_list))
2950 self.add_instances(aa)
2951 else:
2952 self.add_instances(instance_list[:1])
2953 else:
2954 instances = list(filter(lambda tc: tc.platform.default, instance_list))
2955 self.add_instances(instances)
2956
Anas Nashifaff616d2020-04-17 21:24:57 -04002957 for instance in list(filter(lambda inst: not inst.platform.default, instance_list)):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002958 discards[instance] = discards.get(instance, "Not a default test platform")
Anas Nashif9eb9c4c2020-08-26 15:47:25 -04002959 elif emulation_platforms:
2960 self.add_instances(instance_list)
2961 for instance in list(filter(lambda inst: not inst.platform.simulation != 'na', instance_list)):
2962 discards[instance] = discards.get(instance, "Not an emulated platform")
Anas Nashifce2b4182020-03-24 14:40:28 -04002963
2964 else:
2965 self.add_instances(instance_list)
2966
2967 for _, case in self.instances.items():
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02002968 case.create_overlay(case.platform, self.enable_asan, self.enable_ubsan, self.enable_coverage, self.coverage_platform)
Anas Nashifce2b4182020-03-24 14:40:28 -04002969
2970 self.discards = discards
2971 self.selected_platforms = set(p.platform.name for p in self.instances.values())
2972
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002973 for instance in self.discards:
2974 instance.reason = self.discards[instance]
2975 instance.status = "skipped"
2976 instance.fill_results_by_status()
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02002977
Anas Nashifce2b4182020-03-24 14:40:28 -04002978 return discards
2979
2980 def add_instances(self, instance_list):
2981 for instance in instance_list:
2982 self.instances[instance.name] = instance
2983
Anas Nashif405f1b62020-07-27 12:27:13 -04002984 def add_tasks_to_queue(self, build_only=False, test_only=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04002985 for instance in self.instances.values():
Anas Nashif405f1b62020-07-27 12:27:13 -04002986 if build_only:
2987 instance.run = False
2988
Anas Nashifce2b4182020-03-24 14:40:28 -04002989 if test_only:
2990 if instance.run:
2991 pipeline.put({"op": "run", "test": instance, "status": "built"})
2992 else:
Anas Nashifdc43c292020-07-09 09:46:45 -04002993 if instance.status not in ['passed', 'skipped', 'error']:
Anas Nashifce2b4182020-03-24 14:40:28 -04002994 instance.status = None
2995 pipeline.put({"op": "cmake", "test": instance})
2996
2997 return "DONE FEEDING"
2998
2999 def execute(self):
Anas Nashifdc43c292020-07-09 09:46:45 -04003000
Anas Nashifce2b4182020-03-24 14:40:28 -04003001 def calc_one_elf_size(instance):
Anas Nashiff04461e2020-06-29 10:07:02 -04003002 if instance.status not in ["error", "failed", "skipped"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04003003 if instance.platform.type != "native":
3004 size_calc = instance.calculate_sizes()
3005 instance.metrics["ram_size"] = size_calc.get_ram_size()
3006 instance.metrics["rom_size"] = size_calc.get_rom_size()
3007 instance.metrics["unrecognized"] = size_calc.unrecognized_sections()
3008 else:
3009 instance.metrics["ram_size"] = 0
3010 instance.metrics["rom_size"] = 0
3011 instance.metrics["unrecognized"] = []
3012
3013 instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0
3014
3015 logger.info("Adding tasks to the queue...")
3016 # We can use a with statement to ensure threads are cleaned up promptly
3017 with BoundedExecutor(bound=self.jobs, max_workers=self.jobs) as executor:
3018
3019 # start a future for a thread which sends work in through the queue
3020 future_to_test = {
Anas Nashif405f1b62020-07-27 12:27:13 -04003021 executor.submit(self.add_tasks_to_queue, self.build_only, self.test_only): 'FEEDER DONE'}
Anas Nashifce2b4182020-03-24 14:40:28 -04003022
3023 while future_to_test:
3024 # check for status of the futures which are currently working
3025 done, pending = concurrent.futures.wait(future_to_test, timeout=1,
3026 return_when=concurrent.futures.FIRST_COMPLETED)
3027
3028 # if there is incoming work, start a new future
3029 while not pipeline.empty():
3030 # fetch a url from the queue
3031 message = pipeline.get()
3032 test = message['test']
3033
3034 pb = ProjectBuilder(self,
3035 test,
3036 lsan=self.enable_lsan,
3037 asan=self.enable_asan,
Christian Taedcke3dbe9f22020-07-06 16:00:57 +02003038 ubsan=self.enable_ubsan,
Anas Nashifce2b4182020-03-24 14:40:28 -04003039 coverage=self.enable_coverage,
3040 extra_args=self.extra_args,
3041 device_testing=self.device_testing,
3042 cmake_only=self.cmake_only,
3043 cleanup=self.cleanup,
3044 valgrind=self.enable_valgrind,
3045 inline_logs=self.inline_logs,
Anas Nashifce2b4182020-03-24 14:40:28 -04003046 generator=self.generator,
Anas Nashiff6462a32020-03-29 19:02:51 -04003047 generator_cmd=self.generator_cmd,
Anas Nashif50925412020-07-16 17:25:19 -04003048 verbose=self.verbose,
3049 warnings_as_errors=self.warnings_as_errors
Anas Nashifce2b4182020-03-24 14:40:28 -04003050 )
3051 future_to_test[executor.submit(pb.process, message)] = test.name
3052
3053 # process any completed futures
3054 for future in done:
3055 test = future_to_test[future]
3056 try:
3057 data = future.result()
3058 except Exception as exc:
Martí Bolívar9861e5d2020-07-23 10:02:19 -07003059 logger.error('%r generated an exception:' % (test,))
3060 for line in traceback.format_exc().splitlines():
3061 logger.error(line)
Anas Nashifce2b4182020-03-24 14:40:28 -04003062 sys.exit('%r generated an exception: %s' % (test, exc))
3063
3064 else:
3065 if data:
3066 logger.debug(data)
3067
3068 # remove the now completed future
3069 del future_to_test[future]
3070
3071 for future in pending:
3072 test = future_to_test[future]
3073
3074 try:
3075 future.result(timeout=180)
3076 except concurrent.futures.TimeoutError:
3077 logger.warning("{} stuck?".format(test))
3078
3079 if self.enable_size_report and not self.cmake_only:
3080 # Parallelize size calculation
3081 executor = concurrent.futures.ThreadPoolExecutor(self.jobs)
3082 futures = [executor.submit(calc_one_elf_size, instance)
3083 for instance in self.instances.values()]
3084 concurrent.futures.wait(futures)
3085 else:
3086 for instance in self.instances.values():
3087 instance.metrics["ram_size"] = 0
3088 instance.metrics["rom_size"] = 0
3089 instance.metrics["handler_time"] = instance.handler.duration if instance.handler else 0
3090 instance.metrics["unrecognized"] = []
3091
3092 def discard_report(self, filename):
3093
3094 try:
Aastha Groverdcbd9152020-06-16 10:19:51 -07003095 if not self.discards:
Anas Nashifce2b4182020-03-24 14:40:28 -04003096 raise SanityRuntimeError("apply_filters() hasn't been run!")
3097 except Exception as e:
3098 logger.error(str(e))
3099 sys.exit(2)
3100
3101 with open(filename, "wt") as csvfile:
3102 fieldnames = ["test", "arch", "platform", "reason"]
3103 cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
3104 cw.writeheader()
3105 for instance, reason in sorted(self.discards.items()):
3106 rowdict = {"test": instance.testcase.name,
3107 "arch": instance.platform.arch,
3108 "platform": instance.platform.name,
3109 "reason": reason}
3110 cw.writerow(rowdict)
3111
Anas Nashif6915adf2020-04-22 09:39:42 -04003112 def target_report(self, outdir, suffix, append=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04003113 platforms = {inst.platform.name for _, inst in self.instances.items()}
3114 for platform in platforms:
Anas Nashif6915adf2020-04-22 09:39:42 -04003115 if suffix:
3116 filename = os.path.join(outdir,"{}_{}.xml".format(platform, suffix))
3117 else:
3118 filename = os.path.join(outdir,"{}.xml".format(platform))
Anas Nashif90415502020-04-11 22:15:04 -04003119 self.xunit_report(filename, platform, full_report=True, append=append)
Anas Nashifce2b4182020-03-24 14:40:28 -04003120
Anas Nashif90415502020-04-11 22:15:04 -04003121
3122 @staticmethod
3123 def process_log(log_file):
3124 filtered_string = ""
3125 if os.path.exists(log_file):
3126 with open(log_file, "rb") as f:
3127 log = f.read().decode("utf-8")
3128 filtered_string = ''.join(filter(lambda x: x in string.printable, log))
3129
3130 return filtered_string
3131
Anas Nashifa53c8132020-05-05 09:32:46 -04003132
Anas Nashif90415502020-04-11 22:15:04 -04003133 def xunit_report(self, filename, platform=None, full_report=False, append=False):
Anas Nashifa53c8132020-05-05 09:32:46 -04003134 total = 0
3135 if platform:
3136 selected = [platform]
3137 else:
3138 selected = self.selected_platforms
Anas Nashif90415502020-04-11 22:15:04 -04003139
Anas Nashif90415502020-04-11 22:15:04 -04003140 if os.path.exists(filename) and append:
3141 tree = ET.parse(filename)
3142 eleTestsuites = tree.getroot()
Anas Nashif90415502020-04-11 22:15:04 -04003143 else:
Anas Nashifce2b4182020-03-24 14:40:28 -04003144 eleTestsuites = ET.Element('testsuites')
Anas Nashifce2b4182020-03-24 14:40:28 -04003145
Anas Nashifa53c8132020-05-05 09:32:46 -04003146 for p in selected:
3147 inst = self.get_platform_instances(p)
3148 fails = 0
3149 passes = 0
3150 errors = 0
3151 skips = 0
3152 duration = 0
3153
3154 for _, instance in inst.items():
3155 handler_time = instance.metrics.get('handler_time', 0)
3156 duration += handler_time
Anas Nashif405f1b62020-07-27 12:27:13 -04003157 if full_report and instance.run:
Anas Nashifa53c8132020-05-05 09:32:46 -04003158 for k in instance.results.keys():
3159 if instance.results[k] == 'PASS':
3160 passes += 1
3161 elif instance.results[k] == 'BLOCK':
3162 errors += 1
3163 elif instance.results[k] == 'SKIP':
3164 skips += 1
3165 else:
3166 fails += 1
3167 else:
Anas Nashiff04461e2020-06-29 10:07:02 -04003168 if instance.status in ["error", "failed", "timeout"]:
Anas Nashifa53c8132020-05-05 09:32:46 -04003169 if instance.reason in ['build_error', 'handler_crash']:
3170 errors += 1
3171 else:
3172 fails += 1
3173 elif instance.status == 'skipped':
3174 skips += 1
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003175 elif instance.status == 'passed':
Anas Nashifa53c8132020-05-05 09:32:46 -04003176 passes += 1
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003177 else:
3178 logger.error(f"Unknown status {instance.status}")
Anas Nashifa53c8132020-05-05 09:32:46 -04003179
3180 total = (errors + passes + fails + skips)
3181 # do not produce a report if no tests were actually run (only built)
3182 if total == 0:
Anas Nashif90415502020-04-11 22:15:04 -04003183 continue
Anas Nashifce2b4182020-03-24 14:40:28 -04003184
Anas Nashifa53c8132020-05-05 09:32:46 -04003185 run = p
3186 eleTestsuite = None
3187
3188 # When we re-run the tests, we re-use the results and update only with
3189 # the newly run tests.
3190 if os.path.exists(filename) and append:
Anas Nashiff04461e2020-06-29 10:07:02 -04003191 ts = eleTestsuites.findall(f'testsuite/[@name="{p}"]')
3192 if ts:
3193 eleTestsuite = ts[0]
3194 eleTestsuite.attrib['failures'] = "%d" % fails
3195 eleTestsuite.attrib['errors'] = "%d" % errors
Christian Taedckeb2be8042020-08-12 14:21:13 +02003196 eleTestsuite.attrib['skipped'] = "%d" % skips
Anas Nashiff04461e2020-06-29 10:07:02 -04003197 else:
3198 logger.info(f"Did not find any existing results for {p}")
3199 eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite',
3200 name=run, time="%f" % duration,
3201 tests="%d" % (total),
3202 failures="%d" % fails,
Christian Taedckeb2be8042020-08-12 14:21:13 +02003203 errors="%d" % (errors), skipped="%s" % (skips))
Anas Nashiff04461e2020-06-29 10:07:02 -04003204
Anas Nashif90415502020-04-11 22:15:04 -04003205 else:
Anas Nashifa53c8132020-05-05 09:32:46 -04003206 eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite',
3207 name=run, time="%f" % duration,
3208 tests="%d" % (total),
3209 failures="%d" % fails,
Christian Taedckeb2be8042020-08-12 14:21:13 +02003210 errors="%d" % (errors), skipped="%s" % (skips))
Anas Nashif90415502020-04-11 22:15:04 -04003211
Anas Nashifa53c8132020-05-05 09:32:46 -04003212 for _, instance in inst.items():
3213 if full_report:
3214 tname = os.path.basename(instance.testcase.name)
3215 else:
3216 tname = instance.testcase.id
Anas Nashif90415502020-04-11 22:15:04 -04003217
Anas Nashifa53c8132020-05-05 09:32:46 -04003218
3219 handler_time = instance.metrics.get('handler_time', 0)
3220
3221 if full_report:
3222 for k in instance.results.keys():
Anas Nashifa53c8132020-05-05 09:32:46 -04003223 # remove testcases that are being re-run from exiting reports
3224 for tc in eleTestsuite.findall(f'testcase/[@name="{k}"]'):
3225 eleTestsuite.remove(tc)
3226
3227 classname = ".".join(tname.split(".")[:2])
3228 eleTestcase = ET.SubElement(
3229 eleTestsuite, 'testcase',
3230 classname=classname,
3231 name="%s" % (k), time="%f" % handler_time)
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003232
3233 if instance.results[k] in ['FAIL', 'BLOCK'] or \
Anas Nashif405f1b62020-07-27 12:27:13 -04003234 (not instance.run and instance.status in ["error", "failed", "timeout"]):
Anas Nashifa53c8132020-05-05 09:32:46 -04003235 if instance.results[k] == 'FAIL':
3236 el = ET.SubElement(
3237 eleTestcase,
3238 'failure',
3239 type="failure",
3240 message="failed")
3241 else:
3242 el = ET.SubElement(
3243 eleTestcase,
3244 'error',
3245 type="failure",
3246 message="failed")
3247 p = os.path.join(self.outdir, instance.platform.name, instance.testcase.name)
3248 log_file = os.path.join(p, "handler.log")
3249 el.text = self.process_log(log_file)
3250
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003251 elif instance.results[k] == 'PASS' \
Anas Nashif405f1b62020-07-27 12:27:13 -04003252 or (not instance.run and instance.status in ["passed"]):
Anas Nashiff04461e2020-06-29 10:07:02 -04003253 pass
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003254 elif instance.results[k] == 'SKIP' \
Anas Nashif405f1b62020-07-27 12:27:13 -04003255 or (not instance.run and instance.status in ["skipped"]):
Maciej Perkowskie3ff4cf2020-07-17 11:13:50 +02003256 el = ET.SubElement(eleTestcase, 'skipped', type="skipped", message=instance.reason)
Anas Nashiff04461e2020-06-29 10:07:02 -04003257 else:
Anas Nashifce2b4182020-03-24 14:40:28 -04003258 el = ET.SubElement(
3259 eleTestcase,
Anas Nashiff04461e2020-06-29 10:07:02 -04003260 'error',
3261 type="error",
3262 message=f"{instance.reason}")
Anas Nashifa53c8132020-05-05 09:32:46 -04003263 else:
3264 if platform:
3265 classname = ".".join(instance.testcase.name.split(".")[:2])
3266 else:
3267 classname = p + ":" + ".".join(instance.testcase.name.split(".")[:2])
Anas Nashifce2b4182020-03-24 14:40:28 -04003268
Anas Nashiff04461e2020-06-29 10:07:02 -04003269 # remove testcases that are being re-run from exiting reports
3270 for tc in eleTestsuite.findall(f'testcase/[@classname="{classname}"]'):
3271 eleTestsuite.remove(tc)
3272
Anas Nashifa53c8132020-05-05 09:32:46 -04003273 eleTestcase = ET.SubElement(eleTestsuite, 'testcase',
3274 classname=classname,
3275 name="%s" % (instance.testcase.name),
3276 time="%f" % handler_time)
Anas Nashif9e1be4c2020-07-23 08:20:49 -04003277
Anas Nashiff04461e2020-06-29 10:07:02 -04003278 if instance.status in ["error", "failed", "timeout"]:
Anas Nashifa53c8132020-05-05 09:32:46 -04003279 failure = ET.SubElement(
Anas Nashifce2b4182020-03-24 14:40:28 -04003280 eleTestcase,
Anas Nashifa53c8132020-05-05 09:32:46 -04003281 'failure',
3282 type="failure",
Maciej Perkowskib2fa99c2020-05-21 14:45:29 +02003283 message=instance.reason)
Anas Nashiff04461e2020-06-29 10:07:02 -04003284
Anas Nashifa53c8132020-05-05 09:32:46 -04003285 p = ("%s/%s/%s" % (self.outdir, instance.platform.name, instance.testcase.name))
3286 bl = os.path.join(p, "build.log")
3287 hl = os.path.join(p, "handler.log")
3288 log_file = bl
3289 if instance.reason != 'Build error':
3290 if os.path.exists(hl):
3291 log_file = hl
3292 else:
3293 log_file = bl
Anas Nashifce2b4182020-03-24 14:40:28 -04003294
Anas Nashifa53c8132020-05-05 09:32:46 -04003295 failure.text = self.process_log(log_file)
Anas Nashifce2b4182020-03-24 14:40:28 -04003296
Anas Nashifa53c8132020-05-05 09:32:46 -04003297 elif instance.status == "skipped":
3298 ET.SubElement(eleTestcase, 'skipped', type="skipped", message="Skipped")
Anas Nashifce2b4182020-03-24 14:40:28 -04003299
3300 result = ET.tostring(eleTestsuites)
3301 with open(filename, 'wb') as report:
3302 report.write(result)
3303
Anas Nashif1c2f1272020-07-23 09:56:01 -04003304 return fails, passes, errors, skips
Anas Nashif90415502020-04-11 22:15:04 -04003305
Anas Nashifce2b4182020-03-24 14:40:28 -04003306 def csv_report(self, filename):
3307 with open(filename, "wt") as csvfile:
3308 fieldnames = ["test", "arch", "platform", "status",
3309 "extra_args", "handler", "handler_time", "ram_size",
3310 "rom_size"]
3311 cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
3312 cw.writeheader()
3313 for instance in self.instances.values():
3314 rowdict = {"test": instance.testcase.name,
3315 "arch": instance.platform.arch,
3316 "platform": instance.platform.name,
3317 "extra_args": " ".join(instance.testcase.extra_args),
3318 "handler": instance.platform.simulation}
3319
3320 rowdict["status"] = instance.status
Anas Nashiff04461e2020-06-29 10:07:02 -04003321 if instance.status not in ["error", "failed", "timeout"]:
Anas Nashifce2b4182020-03-24 14:40:28 -04003322 if instance.handler:
3323 rowdict["handler_time"] = instance.metrics.get("handler_time", 0)
3324 ram_size = instance.metrics.get("ram_size", 0)
3325 rom_size = instance.metrics.get("rom_size", 0)
3326 rowdict["ram_size"] = ram_size
3327 rowdict["rom_size"] = rom_size
3328 cw.writerow(rowdict)
3329
3330 def get_testcase(self, identifier):
3331 results = []
3332 for _, tc in self.testcases.items():
3333 for case in tc.cases:
3334 if case == identifier:
3335 results.append(tc)
3336 return results
3337
3338
3339class CoverageTool:
3340 """ Base class for every supported coverage tool
3341 """
3342
3343 def __init__(self):
Anas Nashiff6462a32020-03-29 19:02:51 -04003344 self.gcov_tool = None
3345 self.base_dir = None
Anas Nashifce2b4182020-03-24 14:40:28 -04003346
3347 @staticmethod
3348 def factory(tool):
3349 if tool == 'lcov':
Anas Nashiff6462a32020-03-29 19:02:51 -04003350 t = Lcov()
3351 elif tool == 'gcovr':
Marcin Niestroj2652dc72020-08-10 22:27:03 +02003352 t = Gcovr()
Anas Nashiff6462a32020-03-29 19:02:51 -04003353 else:
3354 logger.error("Unsupported coverage tool specified: {}".format(tool))
3355 return None
3356
Anas Nashiff6462a32020-03-29 19:02:51 -04003357 return t
Anas Nashifce2b4182020-03-24 14:40:28 -04003358
3359 @staticmethod
3360 def retrieve_gcov_data(intput_file):
Anas Nashiff6462a32020-03-29 19:02:51 -04003361 logger.debug("Working on %s" % intput_file)
Anas Nashifce2b4182020-03-24 14:40:28 -04003362 extracted_coverage_info = {}
3363 capture_data = False
3364 capture_complete = False
3365 with open(intput_file, 'r') as fp:
3366 for line in fp.readlines():
3367 if re.search("GCOV_COVERAGE_DUMP_START", line):
3368 capture_data = True
3369 continue
3370 if re.search("GCOV_COVERAGE_DUMP_END", line):
3371 capture_complete = True
3372 break
3373 # Loop until the coverage data is found.
3374 if not capture_data:
3375 continue
3376 if line.startswith("*"):
3377 sp = line.split("<")
3378 if len(sp) > 1:
3379 # Remove the leading delimiter "*"
3380 file_name = sp[0][1:]
3381 # Remove the trailing new line char
3382 hex_dump = sp[1][:-1]
3383 else:
3384 continue
3385 else:
3386 continue
3387 extracted_coverage_info.update({file_name: hex_dump})
3388 if not capture_data:
3389 capture_complete = True
3390 return {'complete': capture_complete, 'data': extracted_coverage_info}
3391
3392 @staticmethod
3393 def create_gcda_files(extracted_coverage_info):
Anas Nashiff6462a32020-03-29 19:02:51 -04003394 logger.debug("Generating gcda files")
Anas Nashifce2b4182020-03-24 14:40:28 -04003395 for filename, hexdump_val in extracted_coverage_info.items():
3396 # if kobject_hash is given for coverage gcovr fails
3397 # hence skipping it problem only in gcovr v4.1
3398 if "kobject_hash" in filename:
3399 filename = (filename[:-4]) + "gcno"
3400 try:
3401 os.remove(filename)
3402 except Exception:
3403 pass
3404 continue
3405
3406 with open(filename, 'wb') as fp:
3407 fp.write(bytes.fromhex(hexdump_val))
3408
3409 def generate(self, outdir):
3410 for filename in glob.glob("%s/**/handler.log" % outdir, recursive=True):
3411 gcov_data = self.__class__.retrieve_gcov_data(filename)
3412 capture_complete = gcov_data['complete']
3413 extracted_coverage_info = gcov_data['data']
3414 if capture_complete:
3415 self.__class__.create_gcda_files(extracted_coverage_info)
3416 logger.debug("Gcov data captured: {}".format(filename))
3417 else:
3418 logger.error("Gcov data capture incomplete: {}".format(filename))
3419
3420 with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
3421 ret = self._generate(outdir, coveragelog)
3422 if ret == 0:
3423 logger.info("HTML report generated: {}".format(
3424 os.path.join(outdir, "coverage", "index.html")))
3425
3426
3427class Lcov(CoverageTool):
3428
3429 def __init__(self):
3430 super().__init__()
3431 self.ignores = []
3432
3433 def add_ignore_file(self, pattern):
3434 self.ignores.append('*' + pattern + '*')
3435
3436 def add_ignore_directory(self, pattern):
Marcin Niestrojfc674092020-08-10 23:22:11 +02003437 self.ignores.append('*/' + pattern + '/*')
Anas Nashifce2b4182020-03-24 14:40:28 -04003438
3439 def _generate(self, outdir, coveragelog):
3440 coveragefile = os.path.join(outdir, "coverage.info")
3441 ztestfile = os.path.join(outdir, "ztest.info")
3442 subprocess.call(["lcov", "--gcov-tool", self.gcov_tool,
3443 "--capture", "--directory", outdir,
3444 "--rc", "lcov_branch_coverage=1",
3445 "--output-file", coveragefile], stdout=coveragelog)
3446 # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
3447 subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--extract",
3448 coveragefile,
Anas Nashiff6462a32020-03-29 19:02:51 -04003449 os.path.join(self.base_dir, "tests", "ztest", "*"),
Anas Nashifce2b4182020-03-24 14:40:28 -04003450 "--output-file", ztestfile,
3451 "--rc", "lcov_branch_coverage=1"], stdout=coveragelog)
3452
3453 if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
3454 subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--remove",
3455 ztestfile,
Anas Nashiff6462a32020-03-29 19:02:51 -04003456 os.path.join(self.base_dir, "tests/ztest/test/*"),
Anas Nashifce2b4182020-03-24 14:40:28 -04003457 "--output-file", ztestfile,
3458 "--rc", "lcov_branch_coverage=1"],
3459 stdout=coveragelog)
3460 files = [coveragefile, ztestfile]
3461 else:
3462 files = [coveragefile]
3463
3464 for i in self.ignores:
3465 subprocess.call(
3466 ["lcov", "--gcov-tool", self.gcov_tool, "--remove",
3467 coveragefile, i, "--output-file",
3468 coveragefile, "--rc", "lcov_branch_coverage=1"],
3469 stdout=coveragelog)
3470
3471 # The --ignore-errors source option is added to avoid it exiting due to
3472 # samples/application_development/external_lib/
3473 return subprocess.call(["genhtml", "--legend", "--branch-coverage",
3474 "--ignore-errors", "source",
3475 "-output-directory",
3476 os.path.join(outdir, "coverage")] + files,
3477 stdout=coveragelog)
3478
3479
3480class Gcovr(CoverageTool):
3481
3482 def __init__(self):
3483 super().__init__()
3484 self.ignores = []
3485
3486 def add_ignore_file(self, pattern):
3487 self.ignores.append('.*' + pattern + '.*')
3488
3489 def add_ignore_directory(self, pattern):
Marcin Niestrojfc674092020-08-10 23:22:11 +02003490 self.ignores.append(".*/" + pattern + '/.*')
Anas Nashifce2b4182020-03-24 14:40:28 -04003491
3492 @staticmethod
3493 def _interleave_list(prefix, list):
3494 tuple_list = [(prefix, item) for item in list]
3495 return [item for sublist in tuple_list for item in sublist]
3496
3497 def _generate(self, outdir, coveragelog):
3498 coveragefile = os.path.join(outdir, "coverage.json")
3499 ztestfile = os.path.join(outdir, "ztest.json")
3500
3501 excludes = Gcovr._interleave_list("-e", self.ignores)
3502
3503 # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
Anas Nashiff6462a32020-03-29 19:02:51 -04003504 subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable",
Anas Nashifce2b4182020-03-24 14:40:28 -04003505 self.gcov_tool, "-e", "tests/*"] + excludes +
3506 ["--json", "-o", coveragefile, outdir],
3507 stdout=coveragelog)
3508
Anas Nashiff6462a32020-03-29 19:02:51 -04003509 subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable",
Anas Nashifce2b4182020-03-24 14:40:28 -04003510 self.gcov_tool, "-f", "tests/ztest", "-e",
3511 "tests/ztest/test/*", "--json", "-o", ztestfile,
3512 outdir], stdout=coveragelog)
3513
3514 if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
3515 files = [coveragefile, ztestfile]
3516 else:
3517 files = [coveragefile]
3518
3519 subdir = os.path.join(outdir, "coverage")
3520 os.makedirs(subdir, exist_ok=True)
3521
3522 tracefiles = self._interleave_list("--add-tracefile", files)
3523
Anas Nashiff6462a32020-03-29 19:02:51 -04003524 return subprocess.call(["gcovr", "-r", self.base_dir, "--html",
Anas Nashifce2b4182020-03-24 14:40:28 -04003525 "--html-details"] + tracefiles +
3526 ["-o", os.path.join(subdir, "index.html")],
3527 stdout=coveragelog)
3528class HardwareMap:
3529
3530 schema_path = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk", "hwmap-schema.yaml")
3531
3532 manufacturer = [
3533 'ARM',
3534 'SEGGER',
3535 'MBED',
3536 'STMicroelectronics',
3537 'Atmel Corp.',
3538 'Texas Instruments',
3539 'Silicon Labs',
3540 'NXP Semiconductors',
3541 'Microchip Technology Inc.',
3542 'FTDI',
3543 'Digilent'
3544 ]
3545
3546 runner_mapping = {
3547 'pyocd': [
3548 'DAPLink CMSIS-DAP',
3549 'MBED CMSIS-DAP'
3550 ],
3551 'jlink': [
3552 'J-Link',
3553 'J-Link OB'
3554 ],
3555 'openocd': [
Erwan Gouriou2339fa02020-07-07 17:15:22 +02003556 'STM32 STLink', '^XDS110.*', 'STLINK-V3'
Anas Nashifce2b4182020-03-24 14:40:28 -04003557 ],
3558 'dediprog': [
3559 'TTL232R-3V3',
3560 'MCP2200 USB Serial Port Emulator'
3561 ]
3562 }
3563
3564 def __init__(self):
3565 self.detected = []
3566 self.connected_hardware = []
3567
Watson Zeng0079cec2020-08-25 15:29:43 +08003568 def load_device_from_cmdline(self, serial, platform, pre_script, is_pty):
Anas Nashifce2b4182020-03-24 14:40:28 -04003569 device = {
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +03003570 "serial": None,
Anas Nashifce2b4182020-03-24 14:40:28 -04003571 "platform": platform,
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +03003572 "serial_pty": None,
Anas Nashifce2b4182020-03-24 14:40:28 -04003573 "counter": 0,
3574 "available": True,
Watson Zeng0079cec2020-08-25 15:29:43 +08003575 "connected": True,
3576 "pre_script": pre_script
Anas Nashifce2b4182020-03-24 14:40:28 -04003577 }
Andrei Emeltchenkod8b845b2020-03-31 13:22:30 +03003578
3579 if is_pty:
3580 device['serial_pty'] = serial
3581 else:
3582 device['serial'] = serial
3583
Anas Nashifce2b4182020-03-24 14:40:28 -04003584 self.connected_hardware.append(device)
3585
3586 def load_hardware_map(self, map_file):
3587 hwm_schema = scl.yaml_load(self.schema_path)
3588 self.connected_hardware = scl.yaml_load_verify(map_file, hwm_schema)
3589 for i in self.connected_hardware:
3590 i['counter'] = 0
3591
Martí Bolívar07dce822020-04-13 16:50:51 -07003592 def scan_hw(self, persistent=False):
Anas Nashifce2b4182020-03-24 14:40:28 -04003593 from serial.tools import list_ports
3594
Martí Bolívar07dce822020-04-13 16:50:51 -07003595 if persistent and platform.system() == 'Linux':
3596 # On Linux, /dev/serial/by-id provides symlinks to
3597 # '/dev/ttyACMx' nodes using names which are unique as
3598 # long as manufacturers fill out USB metadata nicely.
3599 #
3600 # This creates a map from '/dev/ttyACMx' device nodes
3601 # to '/dev/serial/by-id/usb-...' symlinks. The symlinks
3602 # go into the hardware map because they stay the same
3603 # even when the user unplugs / replugs the device.
3604 #
3605 # Some inexpensive USB/serial adapters don't result
3606 # in unique names here, though, so use of this feature
3607 # requires explicitly setting persistent=True.
3608 by_id = Path('/dev/serial/by-id')
3609 def readlink(link):
3610 return str((by_id / link).resolve())
3611
3612 persistent_map = {readlink(link): str(link)
3613 for link in by_id.iterdir()}
3614 else:
3615 persistent_map = {}
3616
Anas Nashifce2b4182020-03-24 14:40:28 -04003617 serial_devices = list_ports.comports()
3618 logger.info("Scanning connected hardware...")
3619 for d in serial_devices:
3620 if d.manufacturer in self.manufacturer:
3621
3622 # TI XDS110 can have multiple serial devices for a single board
3623 # assume endpoint 0 is the serial, skip all others
3624 if d.manufacturer == 'Texas Instruments' and not d.location.endswith('0'):
3625 continue
3626 s_dev = {}
3627 s_dev['platform'] = "unknown"
3628 s_dev['id'] = d.serial_number
Martí Bolívar07dce822020-04-13 16:50:51 -07003629 s_dev['serial'] = persistent_map.get(d.device, d.device)
Anas Nashifce2b4182020-03-24 14:40:28 -04003630 s_dev['product'] = d.product
3631 s_dev['runner'] = 'unknown'
3632 for runner, _ in self.runner_mapping.items():
3633 products = self.runner_mapping.get(runner)
3634 if d.product in products:
3635 s_dev['runner'] = runner
3636 continue
3637 # Try regex matching
3638 for p in products:
3639 if re.match(p, d.product):
3640 s_dev['runner'] = runner
3641
3642 s_dev['available'] = True
3643 s_dev['connected'] = True
3644 self.detected.append(s_dev)
3645 else:
3646 logger.warning("Unsupported device (%s): %s" % (d.manufacturer, d))
3647
3648 def write_map(self, hwm_file):
3649 # use existing map
3650 if os.path.exists(hwm_file):
3651 with open(hwm_file, 'r') as yaml_file:
Anas Nashifae61b7e2020-07-06 11:30:55 -04003652 hwm = yaml.load(yaml_file, Loader=SafeLoader)
Øyvind Rønningstad4813f462020-07-01 16:49:38 +02003653 hwm.sort(key=lambda x: x['serial'] or '')
3654
Anas Nashifce2b4182020-03-24 14:40:28 -04003655 # disconnect everything
3656 for h in hwm:
3657 h['connected'] = False
3658 h['serial'] = None
3659
Øyvind Rønningstad4813f462020-07-01 16:49:38 +02003660 self.detected.sort(key=lambda x: x['serial'] or '')
Anas Nashifce2b4182020-03-24 14:40:28 -04003661 for d in self.detected:
3662 for h in hwm:
Øyvind Rønningstad4813f462020-07-01 16:49:38 +02003663 if d['id'] == h['id'] and d['product'] == h['product'] and not h['connected'] and not d.get('match', False):
Anas Nashifce2b4182020-03-24 14:40:28 -04003664 h['connected'] = True
3665 h['serial'] = d['serial']
3666 d['match'] = True
3667
3668 new = list(filter(lambda n: not n.get('match', False), self.detected))
3669 hwm = hwm + new
3670
3671 logger.info("Registered devices:")
3672 self.dump(hwm)
3673
3674 with open(hwm_file, 'w') as yaml_file:
Anas Nashifae61b7e2020-07-06 11:30:55 -04003675 yaml.dump(hwm, yaml_file, Dumper=Dumper, default_flow_style=False)
Anas Nashifce2b4182020-03-24 14:40:28 -04003676
3677 else:
3678 # create new file
3679 with open(hwm_file, 'w') as yaml_file:
Anas Nashifae61b7e2020-07-06 11:30:55 -04003680 yaml.dump(self.detected, yaml_file, Dumper=Dumper, default_flow_style=False)
Anas Nashifce2b4182020-03-24 14:40:28 -04003681 logger.info("Detected devices:")
3682 self.dump(self.detected)
3683
3684 @staticmethod
3685 def dump(hwmap=[], filtered=[], header=[], connected_only=False):
3686 print("")
3687 table = []
3688 if not header:
3689 header = ["Platform", "ID", "Serial device"]
3690 for p in sorted(hwmap, key=lambda i: i['platform']):
3691 platform = p.get('platform')
3692 connected = p.get('connected', False)
3693 if filtered and platform not in filtered:
3694 continue
3695
3696 if not connected_only or connected:
3697 table.append([platform, p.get('id', None), p.get('serial')])
3698
3699 print(tabulate(table, headers=header, tablefmt="github"))
3700
3701
3702def size_report(sc):
3703 logger.info(sc.filename)
3704 logger.info("SECTION NAME VMA LMA SIZE HEX SZ TYPE")
3705 for i in range(len(sc.sections)):
3706 v = sc.sections[i]
3707
3708 logger.info("%-17s 0x%08x 0x%08x %8d 0x%05x %-7s" %
3709 (v["name"], v["virt_addr"], v["load_addr"], v["size"], v["size"],
3710 v["type"]))
3711
3712 logger.info("Totals: %d bytes (ROM), %d bytes (RAM)" %
3713 (sc.rom_size, sc.ram_size))
3714 logger.info("")
3715
3716
3717
3718def export_tests(filename, tests):
3719 with open(filename, "wt") as csvfile:
3720 fieldnames = ['section', 'subsection', 'title', 'reference']
3721 cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
3722 for test in tests:
3723 data = test.split(".")
3724 if len(data) > 1:
3725 subsec = " ".join(data[1].split("_")).title()
3726 rowdict = {
3727 "section": data[0].capitalize(),
3728 "subsection": subsec,
3729 "title": test,
3730 "reference": test
3731 }
3732 cw.writerow(rowdict)
3733 else:
3734 logger.info("{} can't be exported".format(test))