pw_compilation_testing/py/pw_compilation_testing/runner.py - pigweed/pigweed - Git at Google

 # Copyright 2022 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """Executes a compilation failure test."""

 import argparse
 import logging
 from pathlib import Path
 import re
 import string
 import sys
 import subprocess
 from typing import Dict, List, Optional

 import pw_cli.log

 from pw_compilation_testing.generator import Compiler, Expectation, TestCase

 _LOG = logging.getLogger(__package__)

 _RULE_REGEX = re.compile('^rule (?:cxx|.*_cxx)$')
 _NINJA_VARIABLE = re.compile('^([a-zA-Z0-9_]+) = ?')


 # TODO(hepler): Could do this step just once and output the results.
 def find_cc_rule(toolchain_ninja_file: Path) -> Optional[str]:
     """Searches the toolchain.ninja file for the cc rule."""
     cmd_prefix = '  command = '

     found_rule = False

     with toolchain_ninja_file.open() as fd:
         for line in fd:
             if found_rule:
                 if line.startswith(cmd_prefix):
                     cmd = line[len(cmd_prefix):].strip()
                     if cmd.startswith('ccache '):
                         cmd = cmd[len('ccache '):]
                     return cmd

                 if not line.startswith('  '):
                     break
             elif _RULE_REGEX.match(line):
                 found_rule = True

     return None


 def _parse_ninja_variables(target_ninja_file: Path) -> Dict[str, str]:
     variables: Dict[str, str] = {}

     with target_ninja_file.open() as fd:
         for line in fd:
             match = _NINJA_VARIABLE.match(line)
             if match:
                 variables[match.group(1)] = line[match.end():].strip()

     return variables


 _EXPECTED_GN_VARS = (
     'asmflags',
     'cflags',
     'cflags_c',
     'cflags_cc',
     'cflags_objc',
     'cflags_objcc',
     'defines',
     'include_dirs',
 )

 _ENABLE_TEST_MACRO = '-DPW_NC_TEST_EXECUTE_CASE_'
 # Regular expression to find and remove ANSI escape sequences, based on
 # https://stackoverflow.com/a/14693789.
 _ANSI_ESCAPE_SEQUENCES = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')


 class _TestFailure(Exception):
     pass


 def _red(message: str) -> str:
     return f'\033[31m\033[1m{message}\033[0m'


 _TITLE_1 = '     NEGATIVE     '
 _TITLE_2 = ' COMPILATION TEST '

 _BOX_TOP = f'┏{"━" * len(_TITLE_1)}┓'
 _BOX_MID_1 = f'┃{_red(_TITLE_1)}┃ \033[1m{{test_name}}\033[0m'
 _BOX_MID_2 = f'┃{_red(_TITLE_2)}┃ \033[0m{{source}}:{{line}}\033[0m'
 _BOX_BOT = f'┻{"━" * (len(_TITLE_1))}┻{"━" * (77 - len(_TITLE_1))}┓'
 _FOOTER = '\n' + '━' * 79 + '┛'


 def _start_failure(test: TestCase, command: str) -> None:
     print(_BOX_TOP, file=sys.stderr)
     print(_BOX_MID_1.format(test_name=test.name()), file=sys.stderr)
     print(_BOX_MID_2.format(source=test.source, line=test.line),
           file=sys.stderr)
     print(_BOX_BOT, file=sys.stderr)
     print(file=sys.stderr)

     _LOG.debug('Compilation command:\n%s', command)


 def _check_results(test: TestCase, command: str,
                    process: subprocess.CompletedProcess) -> None:
     stderr = process.stderr.decode(errors='replace')

     if process.returncode == 0:
         _start_failure(test, command)
         _LOG.error('Compilation succeeded, but it should have failed!')
         _LOG.error('Update the test code so that is fails to compile.')
         raise _TestFailure

     compiler_str = command.split(' ', 1)[0]
     compiler = Compiler.from_command(compiler_str)

     _LOG.debug('%s is %s', compiler_str, compiler)
     expectations: List[Expectation] = [
         e for e in test.expectations if compiler.matches(e.compiler)
     ]

     _LOG.debug('%s: Checking compilation from %s (%s) for %d of %d patterns:',
                test.name(), compiler_str, compiler, len(expectations),
                len(test.expectations))
     for expectation in expectations:
         _LOG.debug('    %s', expectation.pattern.pattern)

     if not expectations:
         _start_failure(test, command)
         _LOG.error(
             'Compilation with %s failed, but no PW_NC_EXPECT() patterns '
             'that apply to %s were provided', compiler_str, compiler_str)

         _LOG.error('Compilation output:\n%s', stderr)
         _LOG.error('')
         _LOG.error(
             'Add at least one PW_NC_EXPECT("<regex>") or '
             'PW_NC_EXPECT_%s("<regex>") expectation to %s', compiler.name,
             test.case)
         raise _TestFailure

     no_color = _ANSI_ESCAPE_SEQUENCES.sub('', stderr)

     failed = [e for e in expectations if not e.pattern.search(no_color)]
     if failed:
         _start_failure(test, command)
         _LOG.error(
             'Compilation with %s failed, but the output did not '
             'match the expected patterns.', compiler_str)
         _LOG.error('%d of %d expected patterns did not match:', len(failed),
                    len(expectations))
         _LOG.error('')
         for expectation in expectations:
             _LOG.error('  %s %s:%d: %s', '❌' if expectation in failed else '✅',
                        test.source.name, expectation.line,
                        expectation.pattern.pattern)
         _LOG.error('')

         _LOG.error('Compilation output:\n%s', stderr)
         _LOG.error('')
         _LOG.error('Update the test so that compilation fails with the '
                    'expected output')
         raise _TestFailure


 def _execute_test(test: TestCase, command: str, variables: Dict[str, str],
                   all_tests: List[str]) -> None:
     variables['in'] = str(test.source)

     command = string.Template(command).substitute(variables)
     command = ' '.join([
         command,
         '-DPW_NEGATIVE_COMPILATION_TESTS_ENABLED',
         # Define macros to disable all tests except this one.
         *(f'{_ENABLE_TEST_MACRO}{t}={1 if test.case == t else 0}'
           for t in all_tests),
     ])
     process = subprocess.run(command, shell=True, capture_output=True)

     _check_results(test, command, process)


 def _main(test: TestCase, toolchain_ninja: Path, target_ninja: Path,
           all_tests: Path) -> int:
     """Compiles a compile fail test and returns 1 if compilation succeeds."""
     command = find_cc_rule(toolchain_ninja)

     if command is None:
         _LOG.critical('Failed to find C++ compilation command in %s',
                       toolchain_ninja)
         return 2

     variables = {key: '' for key in _EXPECTED_GN_VARS}
     variables.update(_parse_ninja_variables(target_ninja))

     variables['out'] = str(target_ninja.parent /
                            f'{target_ninja.stem}.compile_fail_test.out')

     try:
         _execute_test(test, command, variables,
                       all_tests.read_text().splitlines())
     except _TestFailure:
         print(_FOOTER, file=sys.stderr)
         return 1

     return 0


 def _parse_args() -> dict:
     """Parses command-line arguments."""

     parser = argparse.ArgumentParser(
         description='Emits an error when a facade has a null backend')
     parser.add_argument(
         '--toolchain-ninja',
         type=Path,
         required=True,
         help='Ninja file with the compilation command for the toolchain')
     parser.add_argument(
         '--target-ninja',
         type=Path,
         required=True,
         help='Ninja file with the compilation commands to the test target')
     parser.add_argument('--test-data',
                         dest='test',
                         required=True,
                         type=TestCase.deserialize,
                         help='Serialized TestCase object')
     parser.add_argument('--all-tests', type=Path, help='List of all tests')
     return vars(parser.parse_args())


 if __name__ == '__main__':
     pw_cli.log.install(level=logging.INFO)
     sys.exit(_main(**_parse_args()))
	# Copyright 2022 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""Executes a compilation failure test."""

	import argparse
	import logging
	from pathlib import Path
	import re
	import string
	import sys
	import subprocess
	from typing import Dict, List, Optional

	import pw_cli.log

	from pw_compilation_testing.generator import Compiler, Expectation, TestCase

	_LOG = logging.getLogger(__package__)

	_RULE_REGEX = re.compile('^rule (?:cxx\|.*_cxx)$')
	_NINJA_VARIABLE = re.compile('^([a-zA-Z0-9_]+) = ?')


	# TODO(hepler): Could do this step just once and output the results.
	def find_cc_rule(toolchain_ninja_file: Path) -> Optional[str]:
	"""Searches the toolchain.ninja file for the cc rule."""
	cmd_prefix = ' command = '

	found_rule = False

	with toolchain_ninja_file.open() as fd:
	for line in fd:
	if found_rule:
	if line.startswith(cmd_prefix):
	cmd = line[len(cmd_prefix):].strip()
	if cmd.startswith('ccache '):
	cmd = cmd[len('ccache '):]
	return cmd

	if not line.startswith(' '):
	break
	elif _RULE_REGEX.match(line):
	found_rule = True

	return None


	def _parse_ninja_variables(target_ninja_file: Path) -> Dict[str, str]:
	variables: Dict[str, str] = {}

	with target_ninja_file.open() as fd:
	for line in fd:
	match = _NINJA_VARIABLE.match(line)
	if match:
	variables[match.group(1)] = line[match.end():].strip()

	return variables


	_EXPECTED_GN_VARS = (
	'asmflags',
	'cflags',
	'cflags_c',
	'cflags_cc',
	'cflags_objc',
	'cflags_objcc',
	'defines',
	'include_dirs',
	)

	_ENABLE_TEST_MACRO = '-DPW_NC_TEST_EXECUTE_CASE_'
	# Regular expression to find and remove ANSI escape sequences, based on
	# https://stackoverflow.com/a/14693789.
	_ANSI_ESCAPE_SEQUENCES = re.compile(r'\x1B(?:[@-Z\\-_]\|\[[0-?][ -/][@-~])')


	class _TestFailure(Exception):
	pass


	def _red(message: str) -> str:
	return f'\033[31m\033[1m{message}\033[0m'


	_TITLE_1 = ' NEGATIVE '
	_TITLE_2 = ' COMPILATION TEST '

	_BOX_TOP = f'┏{"━" * len(_TITLE_1)}┓'
	_BOX_MID_1 = f'┃{_red(_TITLE_1)}┃ \033[1m{{test_name}}\033[0m'
	_BOX_MID_2 = f'┃{_red(_TITLE_2)}┃ \033[0m{{source}}:{{line}}\033[0m'
	_BOX_BOT = f'┻{"━" * (len(_TITLE_1))}┻{"━" * (77 - len(_TITLE_1))}┓'
	_FOOTER = '\n' + '━' * 79 + '┛'


	def _start_failure(test: TestCase, command: str) -> None:
	print(_BOX_TOP, file=sys.stderr)
	print(_BOX_MID_1.format(test_name=test.name()), file=sys.stderr)
	print(_BOX_MID_2.format(source=test.source, line=test.line),
	file=sys.stderr)
	print(_BOX_BOT, file=sys.stderr)
	print(file=sys.stderr)

	_LOG.debug('Compilation command:\n%s', command)


	def _check_results(test: TestCase, command: str,
	process: subprocess.CompletedProcess) -> None:
	stderr = process.stderr.decode(errors='replace')

	if process.returncode == 0:
	_start_failure(test, command)
	_LOG.error('Compilation succeeded, but it should have failed!')
	_LOG.error('Update the test code so that is fails to compile.')
	raise _TestFailure

	compiler_str = command.split(' ', 1)[0]
	compiler = Compiler.from_command(compiler_str)

	_LOG.debug('%s is %s', compiler_str, compiler)
	expectations: List[Expectation] = [
	e for e in test.expectations if compiler.matches(e.compiler)
	]

	_LOG.debug('%s: Checking compilation from %s (%s) for %d of %d patterns:',
	test.name(), compiler_str, compiler, len(expectations),
	len(test.expectations))
	for expectation in expectations:
	_LOG.debug(' %s', expectation.pattern.pattern)

	if not expectations:
	_start_failure(test, command)
	_LOG.error(
	'Compilation with %s failed, but no PW_NC_EXPECT() patterns '
	'that apply to %s were provided', compiler_str, compiler_str)

	_LOG.error('Compilation output:\n%s', stderr)
	_LOG.error('')
	_LOG.error(
	'Add at least one PW_NC_EXPECT("<regex>") or '
	'PW_NC_EXPECT_%s("<regex>") expectation to %s', compiler.name,
	test.case)
	raise _TestFailure

	no_color = _ANSI_ESCAPE_SEQUENCES.sub('', stderr)

	failed = [e for e in expectations if not e.pattern.search(no_color)]
	if failed:
	_start_failure(test, command)
	_LOG.error(
	'Compilation with %s failed, but the output did not '
	'match the expected patterns.', compiler_str)
	_LOG.error('%d of %d expected patterns did not match:', len(failed),
	len(expectations))
	_LOG.error('')
	for expectation in expectations:
	_LOG.error(' %s %s:%d: %s', '❌' if expectation in failed else '✅',
	test.source.name, expectation.line,
	expectation.pattern.pattern)
	_LOG.error('')

	_LOG.error('Compilation output:\n%s', stderr)
	_LOG.error('')
	_LOG.error('Update the test so that compilation fails with the '
	'expected output')
	raise _TestFailure


	def _execute_test(test: TestCase, command: str, variables: Dict[str, str],
	all_tests: List[str]) -> None:
	variables['in'] = str(test.source)

	command = string.Template(command).substitute(variables)
	command = ' '.join([
	command,
	'-DPW_NEGATIVE_COMPILATION_TESTS_ENABLED',
	# Define macros to disable all tests except this one.
	*(f'{_ENABLE_TEST_MACRO}{t}={1 if test.case == t else 0}'
	for t in all_tests),
	])
	process = subprocess.run(command, shell=True, capture_output=True)

	_check_results(test, command, process)


	def _main(test: TestCase, toolchain_ninja: Path, target_ninja: Path,
	all_tests: Path) -> int:
	"""Compiles a compile fail test and returns 1 if compilation succeeds."""
	command = find_cc_rule(toolchain_ninja)

	if command is None:
	_LOG.critical('Failed to find C++ compilation command in %s',
	toolchain_ninja)
	return 2

	variables = {key: '' for key in _EXPECTED_GN_VARS}
	variables.update(_parse_ninja_variables(target_ninja))

	variables['out'] = str(target_ninja.parent /
	f'{target_ninja.stem}.compile_fail_test.out')

	try:
	_execute_test(test, command, variables,
	all_tests.read_text().splitlines())
	except _TestFailure:
	print(_FOOTER, file=sys.stderr)
	return 1

	return 0


	def _parse_args() -> dict:
	"""Parses command-line arguments."""

	parser = argparse.ArgumentParser(
	description='Emits an error when a facade has a null backend')
	parser.add_argument(
	'--toolchain-ninja',
	type=Path,
	required=True,
	help='Ninja file with the compilation command for the toolchain')
	parser.add_argument(
	'--target-ninja',
	type=Path,
	required=True,
	help='Ninja file with the compilation commands to the test target')
	parser.add_argument('--test-data',
	dest='test',
	required=True,
	type=TestCase.deserialize,
	help='Serialized TestCase object')
	parser.add_argument('--all-tests', type=Path, help='List of all tests')
	return vars(parser.parse_args())


	if __name__ == '__main__':
	pw_cli.log.install(level=logging.INFO)
	sys.exit(_main(**_parse_args()))