pw_arduino_build/py/pw_arduino_build/unit_test_runner.py - pigweed/pigweed - Git at Google

 #!/usr/bin/env python3
 # Copyright 2020 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """This script flashes and runs unit tests onto Arduino boards."""

 import argparse
 import logging
 import os
 import platform
 import re
 import subprocess
 import sys
 import time
 from pathlib import Path
 from typing import List

 import serial  # type: ignore
 import serial.tools.list_ports  # type: ignore
 import pw_arduino_build.log
 from pw_arduino_build import teensy_detector
 from pw_arduino_build.file_operations import decode_file_json

 _LOG = logging.getLogger('unit_test_runner')

 # Verification of test pass/failure depends on these strings. If the formatting
 # or output of the simple_printing_event_handler changes, this may need to be
 # updated.
 _TESTS_STARTING_STRING = b'[==========] Running all tests.'
 _TESTS_DONE_STRING = b'[==========] Done running all tests.'
 _TEST_FAILURE_STRING = b'[  FAILED  ]'

 # How long to wait for the first byte of a test to be emitted. This is longer
 # than the user-configurable timeout as there's a delay while the device is
 # flashed.
 _FLASH_TIMEOUT = 5.0


 class TestingFailure(Exception):
     """A simple exception to be raised when a testing step fails."""


 class DeviceNotFound(Exception):
     """A simple exception to be raised when unable to connect to a device."""


 class ArduinoCoreNotSupported(Exception):
     """Exception raised when a given core does not support unit testing."""


 def valid_file_name(arg):
     file_path = Path(os.path.expandvars(arg)).absolute()
     if not file_path.is_file():
         raise argparse.ArgumentTypeError(f"'{arg}' does not exist.")
     return file_path


 def parse_args():
     """Parses command-line arguments."""

     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('binary',
                         help='The target test binary to run',
                         type=valid_file_name)
     parser.add_argument('--port',
                         help='The name of the serial port to connect to when '
                         'running tests')
     parser.add_argument('--baud',
                         type=int,
                         default=115200,
                         help='Target baud rate to use for serial communication'
                         ' with target device')
     parser.add_argument('--test-timeout',
                         type=float,
                         default=5.0,
                         help='Maximum communication delay in seconds before a '
                         'test is considered unresponsive and aborted')
     parser.add_argument('--verbose',
                         '-v',
                         dest='verbose',
                         action='store_true',
                         help='Output additional logs as the script runs')

     parser.add_argument('--flash-only',
                         action='store_true',
                         help="Don't check for test output after flashing.")

     # arduino_builder arguments
     # TODO(tonymd): Get these args from __main__.py or elsewhere.
     parser.add_argument("-c",
                         "--config-file",
                         required=True,
                         help="Path to a config file.")
     parser.add_argument("--arduino-package-path",
                         help="Path to the arduino IDE install location.")
     parser.add_argument("--arduino-package-name",
                         help="Name of the Arduino board package to use.")
     parser.add_argument("--compiler-path-override",
                         help="Path to arm-none-eabi-gcc bin folder. "
                         "Default: Arduino core specified gcc")
     parser.add_argument("--board", help="Name of the Arduino board to use.")
     parser.add_argument("--upload-tool",
                         required=True,
                         help="Name of the Arduino upload tool to use.")
     parser.add_argument("--set-variable",
                         action="append",
                         metavar='some.variable=NEW_VALUE',
                         help="Override an Arduino recipe variable. May be "
                         "specified multiple times. For example: "
                         "--set-variable 'serial.port.label=/dev/ttyACM0' "
                         "--set-variable 'serial.port.protocol=Teensy'")
     return parser.parse_args()


 def log_subprocess_output(level, output):
     """Logs subprocess output line-by-line."""

     lines = output.decode('utf-8', errors='replace').splitlines()
     for line in lines:
         _LOG.log(level, line)


 def read_serial(port, baud_rate, test_timeout) -> bytes:
     """Reads lines from a serial port until a line read times out.

     Returns bytes object containing the read serial data.
     """

     serial_data = bytearray()
     device = serial.Serial(baudrate=baud_rate,
                            port=port,
                            timeout=_FLASH_TIMEOUT)
     if not device.is_open:
         raise TestingFailure('Failed to open device')

     # Flush input buffer and reset the device to begin the test.
     device.reset_input_buffer()

     # Block and wait for the first byte.
     serial_data += device.read()
     if not serial_data:
         raise TestingFailure('Device not producing output')

     device.timeout = test_timeout

     # Read with a reasonable timeout until we stop getting characters.
     while True:
         bytes_read = device.readline()
         if not bytes_read:
             break
         serial_data += bytes_read
         if serial_data.rfind(_TESTS_DONE_STRING) != -1:
             # Set to much more aggressive timeout since the last one or two
             # lines should print out immediately. (one line if all fails or all
             # passes, two lines if mixed.)
             device.timeout = 0.01

     # Remove carriage returns.
     serial_data = serial_data.replace(b'\r', b'')

     # Try to trim captured results to only contain most recent test run.
     test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
     return serial_data if test_start_index == -1 else serial_data[
         test_start_index:]


 def wait_for_port(port):
     """Wait for the serial port to be available."""
     while port not in [sp.device for sp in serial.tools.list_ports.comports()]:
         time.sleep(1)


 def flash_device(test_runner_args, upload_tool):
     """Flash binary to a connected device using the provided configuration."""

     # TODO(tonymd): Create a library function to call rather than launching
     # the arduino_builder script.
     flash_tool = 'arduino_builder'
     cmd = [flash_tool, "--quiet"] + test_runner_args + [
         "--run-objcopy", "--run-postbuilds", "--run-upload", upload_tool
     ]
     _LOG.info('Flashing firmware to device')
     _LOG.debug('Running: %s', " ".join(cmd))

     env = os.environ.copy()
     process = subprocess.run(cmd,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
                              env=env)
     if process.returncode:
         log_subprocess_output(logging.ERROR, process.stdout)
         raise TestingFailure('Failed to flash target device')

     log_subprocess_output(logging.DEBUG, process.stdout)

     _LOG.debug('Successfully flashed firmware to device')


 def handle_test_results(test_output):
     """Parses test output to determine whether tests passed or failed."""

     if test_output.find(_TESTS_STARTING_STRING) == -1:
         raise TestingFailure('Failed to find test start')

     if test_output.rfind(_TESTS_DONE_STRING) == -1:
         log_subprocess_output(logging.INFO, test_output)
         raise TestingFailure('Tests did not complete')

     if test_output.rfind(_TEST_FAILURE_STRING) != -1:
         log_subprocess_output(logging.INFO, test_output)
         raise TestingFailure('Test suite had one or more failures')

     log_subprocess_output(logging.DEBUG, test_output)

     _LOG.info('Test passed!')


 def run_device_test(binary, flash_only, port, baud, test_timeout, upload_tool,
                     arduino_package_path, test_runner_args) -> bool:
     """Flashes, runs, and checks an on-device test binary.

     Returns true on test pass.
     """
     if test_runner_args is None:
         test_runner_args = []

     if "teensy" not in arduino_package_path:
         raise ArduinoCoreNotSupported(arduino_package_path)

     if port is None or "--set-variable" not in test_runner_args:
         _LOG.debug('Attempting to automatically detect dev board')
         boards = teensy_detector.detect_boards(arduino_package_path)
         if not boards:
             error = 'Could not find an attached device'
             _LOG.error(error)
             raise DeviceNotFound(error)
         test_runner_args += boards[0].test_runner_args()
         upload_tool = boards[0].arduino_upload_tool_name
         if port is None:
             port = boards[0].dev_name

     # TODO(tonymd): Remove this when teensy_ports is working in teensy_detector
     if platform.system() == "Windows":
         # Delete the incorrect serial port.
         index_of_port = [
             i for i, l in enumerate(test_runner_args)
             if l.startswith('serial.port=')
         ]
         if index_of_port:
             # Delete the '--set-variable' arg
             del test_runner_args[index_of_port[0] - 1]
             # Delete the 'serial.port=*' arg
             del test_runner_args[index_of_port[0] - 1]

     _LOG.debug('Launching test binary %s', binary)
     try:
         result: List[bytes] = []
         _LOG.info('Running test')
         # Warning: A race condition is possible here. This assumes the host is
         # able to connect to the port and that there isn't a test running on
         # this serial port.
         flash_device(test_runner_args, upload_tool)
         wait_for_port(port)
         if flash_only:
             return True
         result.append(read_serial(port, baud, test_timeout))
         if result:
             handle_test_results(result[0])
     except TestingFailure as err:
         _LOG.error(err)
         return False

     return True


 def get_option(key, config_file_values, args, required=False):
     command_line_option = getattr(args, key, None)
     final_option = config_file_values.get(key, command_line_option)
     if required and command_line_option is None and final_option is None:
         # Print a similar error message to argparse
         executable = os.path.basename(sys.argv[0])
         option = "--" + key.replace("_", "-")
         print(f"{executable}: error: the following arguments are required: "
               f"{option}")
         sys.exit(1)
     return final_option


 def main():
     """Set up runner, and then flash/run device test."""
     args = parse_args()

     json_file_options, unused_config_path = decode_file_json(args.config_file)

     log_level = logging.DEBUG if args.verbose else logging.INFO
     pw_arduino_build.log.install(log_level)

     # Construct arduino_builder flash arguments for a given .elf binary.
     arduino_package_path = get_option("arduino_package_path",
                                       json_file_options,
                                       args,
                                       required=True)
     # Arduino core args.
     arduino_builder_args = [
         "--arduino-package-path",
         arduino_package_path,
         "--arduino-package-name",
         get_option("arduino_package_name",
                    json_file_options,
                    args,
                    required=True),
     ]

     # Use CIPD installed compilers.
     compiler_path_override = get_option("compiler_path_override",
                                         json_file_options, args)
     if compiler_path_override:
         arduino_builder_args += [
             "--compiler-path-override", compiler_path_override
         ]

     # Run subcommand with board selection arg.
     arduino_builder_args += [
         "run", "--board",
         get_option("board", json_file_options, args, required=True)
     ]

     # .elf file location args.
     binary = args.binary
     build_path = binary.parent.as_posix()
     arduino_builder_args += ["--build-path", build_path]
     build_project_name = binary.name
     # Remove '.elf' extension.
     match_result = re.match(r'(.*?)\.elf$', binary.name, re.IGNORECASE)
     if match_result:
         build_project_name = match_result[1]
         arduino_builder_args += ["--build-project-name", build_project_name]

     # USB port is passed to arduino_builder_args via --set-variable args.
     if args.set_variable:
         for var in args.set_variable:
             arduino_builder_args += ["--set-variable", var]

     if run_device_test(binary.as_posix(),
                        args.flash_only,
                        args.port,
                        args.baud,
                        args.test_timeout,
                        args.upload_tool,
                        arduino_package_path,
                        test_runner_args=arduino_builder_args):
         sys.exit(0)
     else:
         sys.exit(1)


 if __name__ == '__main__':
     main()
	#!/usr/bin/env python3
	# Copyright 2020 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""This script flashes and runs unit tests onto Arduino boards."""

	import argparse
	import logging
	import os
	import platform
	import re
	import subprocess
	import sys
	import time
	from pathlib import Path
	from typing import List

	import serial # type: ignore
	import serial.tools.list_ports # type: ignore
	import pw_arduino_build.log
	from pw_arduino_build import teensy_detector
	from pw_arduino_build.file_operations import decode_file_json

	_LOG = logging.getLogger('unit_test_runner')

	# Verification of test pass/failure depends on these strings. If the formatting
	# or output of the simple_printing_event_handler changes, this may need to be
	# updated.
	_TESTS_STARTING_STRING = b'[==========] Running all tests.'
	_TESTS_DONE_STRING = b'[==========] Done running all tests.'
	_TEST_FAILURE_STRING = b'[ FAILED ]'

	# How long to wait for the first byte of a test to be emitted. This is longer
	# than the user-configurable timeout as there's a delay while the device is
	# flashed.
	_FLASH_TIMEOUT = 5.0


	class TestingFailure(Exception):
	"""A simple exception to be raised when a testing step fails."""


	class DeviceNotFound(Exception):
	"""A simple exception to be raised when unable to connect to a device."""


	class ArduinoCoreNotSupported(Exception):
	"""Exception raised when a given core does not support unit testing."""


	def valid_file_name(arg):
	file_path = Path(os.path.expandvars(arg)).absolute()
	if not file_path.is_file():
	raise argparse.ArgumentTypeError(f"'{arg}' does not exist.")
	return file_path


	def parse_args():
	"""Parses command-line arguments."""

	parser = argparse.ArgumentParser(description=__doc__)
	parser.add_argument('binary',
	help='The target test binary to run',
	type=valid_file_name)
	parser.add_argument('--port',
	help='The name of the serial port to connect to when '
	'running tests')
	parser.add_argument('--baud',
	type=int,
	default=115200,
	help='Target baud rate to use for serial communication'
	' with target device')
	parser.add_argument('--test-timeout',
	type=float,
	default=5.0,
	help='Maximum communication delay in seconds before a '
	'test is considered unresponsive and aborted')
	parser.add_argument('--verbose',
	'-v',
	dest='verbose',
	action='store_true',
	help='Output additional logs as the script runs')

	parser.add_argument('--flash-only',
	action='store_true',
	help="Don't check for test output after flashing.")

	# arduino_builder arguments
	# TODO(tonymd): Get these args from __main__.py or elsewhere.
	parser.add_argument("-c",
	"--config-file",
	required=True,
	help="Path to a config file.")
	parser.add_argument("--arduino-package-path",
	help="Path to the arduino IDE install location.")
	parser.add_argument("--arduino-package-name",
	help="Name of the Arduino board package to use.")
	parser.add_argument("--compiler-path-override",
	help="Path to arm-none-eabi-gcc bin folder. "
	"Default: Arduino core specified gcc")
	parser.add_argument("--board", help="Name of the Arduino board to use.")
	parser.add_argument("--upload-tool",
	required=True,
	help="Name of the Arduino upload tool to use.")
	parser.add_argument("--set-variable",
	action="append",
	metavar='some.variable=NEW_VALUE',
	help="Override an Arduino recipe variable. May be "
	"specified multiple times. For example: "
	"--set-variable 'serial.port.label=/dev/ttyACM0' "
	"--set-variable 'serial.port.protocol=Teensy'")
	return parser.parse_args()


	def log_subprocess_output(level, output):
	"""Logs subprocess output line-by-line."""

	lines = output.decode('utf-8', errors='replace').splitlines()
	for line in lines:
	_LOG.log(level, line)


	def read_serial(port, baud_rate, test_timeout) -> bytes:
	"""Reads lines from a serial port until a line read times out.

	Returns bytes object containing the read serial data.
	"""

	serial_data = bytearray()
	device = serial.Serial(baudrate=baud_rate,
	port=port,
	timeout=_FLASH_TIMEOUT)
	if not device.is_open:
	raise TestingFailure('Failed to open device')

	# Flush input buffer and reset the device to begin the test.
	device.reset_input_buffer()

	# Block and wait for the first byte.
	serial_data += device.read()
	if not serial_data:
	raise TestingFailure('Device not producing output')

	device.timeout = test_timeout

	# Read with a reasonable timeout until we stop getting characters.
	while True:
	bytes_read = device.readline()
	if not bytes_read:
	break
	serial_data += bytes_read
	if serial_data.rfind(_TESTS_DONE_STRING) != -1:
	# Set to much more aggressive timeout since the last one or two
	# lines should print out immediately. (one line if all fails or all
	# passes, two lines if mixed.)
	device.timeout = 0.01

	# Remove carriage returns.
	serial_data = serial_data.replace(b'\r', b'')

	# Try to trim captured results to only contain most recent test run.
	test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
	return serial_data if test_start_index == -1 else serial_data[
	test_start_index:]


	def wait_for_port(port):
	"""Wait for the serial port to be available."""
	while port not in [sp.device for sp in serial.tools.list_ports.comports()]:
	time.sleep(1)


	def flash_device(test_runner_args, upload_tool):
	"""Flash binary to a connected device using the provided configuration."""

	# TODO(tonymd): Create a library function to call rather than launching
	# the arduino_builder script.
	flash_tool = 'arduino_builder'
	cmd = [flash_tool, "--quiet"] + test_runner_args + [
	"--run-objcopy", "--run-postbuilds", "--run-upload", upload_tool
	]
	_LOG.info('Flashing firmware to device')
	_LOG.debug('Running: %s', " ".join(cmd))

	env = os.environ.copy()
	process = subprocess.run(cmd,
	stdout=subprocess.PIPE,
	stderr=subprocess.STDOUT,
	env=env)
	if process.returncode:
	log_subprocess_output(logging.ERROR, process.stdout)
	raise TestingFailure('Failed to flash target device')

	log_subprocess_output(logging.DEBUG, process.stdout)

	_LOG.debug('Successfully flashed firmware to device')


	def handle_test_results(test_output):
	"""Parses test output to determine whether tests passed or failed."""

	if test_output.find(_TESTS_STARTING_STRING) == -1:
	raise TestingFailure('Failed to find test start')

	if test_output.rfind(_TESTS_DONE_STRING) == -1:
	log_subprocess_output(logging.INFO, test_output)
	raise TestingFailure('Tests did not complete')

	if test_output.rfind(_TEST_FAILURE_STRING) != -1:
	log_subprocess_output(logging.INFO, test_output)
	raise TestingFailure('Test suite had one or more failures')

	log_subprocess_output(logging.DEBUG, test_output)

	_LOG.info('Test passed!')


	def run_device_test(binary, flash_only, port, baud, test_timeout, upload_tool,
	arduino_package_path, test_runner_args) -> bool:
	"""Flashes, runs, and checks an on-device test binary.

	Returns true on test pass.
	"""
	if test_runner_args is None:
	test_runner_args = []

	if "teensy" not in arduino_package_path:
	raise ArduinoCoreNotSupported(arduino_package_path)

	if port is None or "--set-variable" not in test_runner_args:
	_LOG.debug('Attempting to automatically detect dev board')
	boards = teensy_detector.detect_boards(arduino_package_path)
	if not boards:
	error = 'Could not find an attached device'
	_LOG.error(error)
	raise DeviceNotFound(error)
	test_runner_args += boards[0].test_runner_args()
	upload_tool = boards[0].arduino_upload_tool_name
	if port is None:
	port = boards[0].dev_name

	# TODO(tonymd): Remove this when teensy_ports is working in teensy_detector
	if platform.system() == "Windows":
	# Delete the incorrect serial port.
	index_of_port = [
	i for i, l in enumerate(test_runner_args)
	if l.startswith('serial.port=')
	]
	if index_of_port:
	# Delete the '--set-variable' arg
	del test_runner_args[index_of_port[0] - 1]
	# Delete the 'serial.port=*' arg
	del test_runner_args[index_of_port[0] - 1]

	_LOG.debug('Launching test binary %s', binary)
	try:
	result: List[bytes] = []
	_LOG.info('Running test')
	# Warning: A race condition is possible here. This assumes the host is
	# able to connect to the port and that there isn't a test running on
	# this serial port.
	flash_device(test_runner_args, upload_tool)
	wait_for_port(port)
	if flash_only:
	return True
	result.append(read_serial(port, baud, test_timeout))
	if result:
	handle_test_results(result[0])
	except TestingFailure as err:
	_LOG.error(err)
	return False

	return True


	def get_option(key, config_file_values, args, required=False):
	command_line_option = getattr(args, key, None)
	final_option = config_file_values.get(key, command_line_option)
	if required and command_line_option is None and final_option is None:
	# Print a similar error message to argparse
	executable = os.path.basename(sys.argv[0])
	option = "--" + key.replace("_", "-")
	print(f"{executable}: error: the following arguments are required: "
	f"{option}")
	sys.exit(1)
	return final_option


	def main():
	"""Set up runner, and then flash/run device test."""
	args = parse_args()

	json_file_options, unused_config_path = decode_file_json(args.config_file)

	log_level = logging.DEBUG if args.verbose else logging.INFO
	pw_arduino_build.log.install(log_level)

	# Construct arduino_builder flash arguments for a given .elf binary.
	arduino_package_path = get_option("arduino_package_path",
	json_file_options,
	args,
	required=True)
	# Arduino core args.
	arduino_builder_args = [
	"--arduino-package-path",
	arduino_package_path,
	"--arduino-package-name",
	get_option("arduino_package_name",
	json_file_options,
	args,
	required=True),
	]

	# Use CIPD installed compilers.
	compiler_path_override = get_option("compiler_path_override",
	json_file_options, args)
	if compiler_path_override:
	arduino_builder_args += [
	"--compiler-path-override", compiler_path_override
	]

	# Run subcommand with board selection arg.
	arduino_builder_args += [
	"run", "--board",
	get_option("board", json_file_options, args, required=True)
	]

	# .elf file location args.
	binary = args.binary
	build_path = binary.parent.as_posix()
	arduino_builder_args += ["--build-path", build_path]
	build_project_name = binary.name
	# Remove '.elf' extension.
	match_result = re.match(r'(.*?)\.elf$', binary.name, re.IGNORECASE)
	if match_result:
	build_project_name = match_result[1]
	arduino_builder_args += ["--build-project-name", build_project_name]

	# USB port is passed to arduino_builder_args via --set-variable args.
	if args.set_variable:
	for var in args.set_variable:
	arduino_builder_args += ["--set-variable", var]

	if run_device_test(binary.as_posix(),
	args.flash_only,
	args.port,
	args.baud,
	args.test_timeout,
	args.upload_tool,
	arduino_package_path,
	test_runner_args=arduino_builder_args):
	sys.exit(0)
	else:
	sys.exit(1)


	if __name__ == '__main__':
	main()