scripts/tests/twister_blackbox/test_footprint.py - third_party/github/zephyrproject-rtos/zephyr - Git at Google

 #!/usr/bin/env python3
 # Copyright (c) 2024 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 """
 Blackbox tests for twister's command line functions related to memory footprints.
 """

 import importlib
 import json
 import mock
 import os
 import pytest
 import sys

 from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test
 from twisterlib.testplan import TestPlan


 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
 class TestFootprint:
     # Log printed when entering delta calculations
     FOOTPRINT_LOG = 'running footprint_reports'

     # These warnings notify us that deltas were shown in log.
     # Coupled with the code under test.
     DELTA_WARNING_RELEASE = 'Deltas based on metrics from last release'
     DELTA_WARNING_RUN = 'Deltas based on metrics from last run'

     # Size report key we modify to control for deltas
     RAM_KEY = 'used_ram'

     @classmethod
     def setup_class(cls):
         apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
         cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
         cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
         cls.twister_module = importlib.util.module_from_spec(cls.spec)

     @classmethod
     def teardown_class(cls):
         pass

     @pytest.mark.parametrize(
         'old_ram_multiplier, expect_delta_log',
         [
             (0.75, True),
             (1.25, False),
         ],
         ids=['footprint increased', 'footprint reduced']
     )
     def test_compare_report(self, caplog, out_path, old_ram_multiplier, expect_delta_log):
         # First run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--enable-size-report'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Modify the older report so we can control the difference.
         # Note: if footprint tests take too long, replace first run with a prepared twister.json
         # That will increase test-to-code_under_test coupling, however.
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
             f.write(json.dumps(j, indent=4))

         report_path = os.path.join(
             os.path.dirname(out_path),
             f'{os.path.basename(out_path)}.1',
             'twister.json'
         )

         # Second run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--compare-report', report_path] + \
                ['--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         assert self.FOOTPRINT_LOG in caplog.text

         if expect_delta_log:
             assert self.RAM_KEY in caplog.text
             assert self.DELTA_WARNING_RELEASE in caplog.text, \
                 'Expected footprint deltas not logged.'
         else:
             assert self.RAM_KEY not in caplog.text
             assert self.DELTA_WARNING_RELEASE not in caplog.text, \
                 'Unexpected footprint deltas logged.'

     def test_footprint_from_buildlog(self, out_path):
         # First run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                [] + \
                ['--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Get values
         old_values = []
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     assert self.RAM_KEY in ts
                     old_values += [ts[self.RAM_KEY]]

         # Second run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--footprint-from-buildlog'] + \
                ['--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Get values
         new_values = []
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     assert self.RAM_KEY in ts
                     new_values += [ts[self.RAM_KEY]]

         # There can be false positives if our calculations become too accurate.
         # Turn this test into a dummy (check only exit value) in such case.
         assert sorted(old_values) != sorted(new_values), \
             'Same values whether calculating size or using buildlog.'

     @pytest.mark.parametrize(
         'old_ram_multiplier, threshold, expect_delta_log',
         [
             (0.75, 95, False),
             (0.75, 25, True),
         ],
         ids=['footprint threshold not met', 'footprint threshold met']
     )
     def test_footprint_threshold(self, caplog, out_path, old_ram_multiplier,
                                  threshold, expect_delta_log):
         # First run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--enable-size-report'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Modify the older report so we can control the difference.
         # Note: if footprint tests take too long, replace first run with a prepared twister.json
         # That will increase test-to-code_under_test coupling, however.
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
             f.write(json.dumps(j, indent=4))

         report_path = os.path.join(
             os.path.dirname(out_path),
             f'{os.path.basename(out_path)}.1',
             'twister.json'
         )

         # Second run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                [f'--footprint-threshold={threshold}'] + \
                ['--compare-report', report_path, '--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         assert self.FOOTPRINT_LOG in caplog.text

         if expect_delta_log:
             assert self.RAM_KEY in caplog.text
             assert self.DELTA_WARNING_RELEASE in caplog.text, \
                 'Expected footprint deltas not logged.'
         else:
             assert self.RAM_KEY not in caplog.text
             assert self.DELTA_WARNING_RELEASE not in caplog.text, \
                 'Unexpected footprint deltas logged.'

     @pytest.mark.parametrize(
         'flags, old_ram_multiplier, expect_delta_log',
         [
             ([], 0.75, False),
             (['--show-footprint'], 0.75, True),
         ],
         ids=['footprint reduced, no show', 'footprint reduced, show']
     )
     def test_show_footprint(self, caplog, out_path, flags, old_ram_multiplier, expect_delta_log):
         # First run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--enable-size-report'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Modify the older report so we can control the difference.
         # Note: if footprint tests take too long, replace first run with a prepared twister.json
         # That will increase test-to-code_under_test coupling, however.
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
             f.write(json.dumps(j, indent=4))

         report_path = os.path.join(
             os.path.dirname(out_path),
             f'{os.path.basename(out_path)}.1',
             'twister.json'
         )

         # Second run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                flags + \
                ['--compare-report', report_path] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
         print(args)
         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         assert self.FOOTPRINT_LOG in caplog.text

         if expect_delta_log:
             assert self.RAM_KEY in caplog.text
             assert self.DELTA_WARNING_RELEASE in caplog.text, \
                 'Expected footprint deltas not logged.'
         else:
             assert self.RAM_KEY not in caplog.text
             assert self.DELTA_WARNING_RELEASE not in caplog.text, \
                 'Unexpected footprint deltas logged.'

     @pytest.mark.parametrize(
         'old_ram_multiplier, expect_delta_log',
         [
             (0.75, True),
             (1.25, False),
         ],
         ids=['footprint increased', 'footprint reduced']
     )
     def test_last_metrics(self, caplog, out_path, old_ram_multiplier, expect_delta_log):
         # First run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--enable-size-report'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Modify the older report so we can control the difference.
         # Note: if footprint tests take too long, replace first run with a prepared twister.json
         # That will increase test-to-code_under_test coupling, however.
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
             f.write(json.dumps(j, indent=4))

         report_path = os.path.join(
             os.path.dirname(out_path),
             f'{os.path.basename(out_path)}.1',
             'twister.json'
         )

         # Second run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--last-metrics'] + \
                ['--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         assert self.FOOTPRINT_LOG in caplog.text

         if expect_delta_log:
             assert self.RAM_KEY in caplog.text
             assert self.DELTA_WARNING_RUN in caplog.text, \
                 'Expected footprint deltas not logged.'
         else:
             assert self.RAM_KEY not in caplog.text
             assert self.DELTA_WARNING_RUN not in caplog.text, \
                 'Unexpected footprint deltas logged.'

         second_logs = caplog.records
         caplog.clear()
         clear_log_in_test()

         # Third run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--compare-report', report_path] + \
                ['--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Since second run should use the same source as the third, we should compare them.
         delta_logs = [
             record.getMessage() for record in second_logs \
             if self.RAM_KEY in record.getMessage()
         ]
         assert all([log in caplog.text for log in delta_logs])

     @pytest.mark.parametrize(
         'old_ram_multiplier, expect_delta_log',
         [
             (0.75, True),
             (1.00, False),
             (1.25, True),
         ],
         ids=['footprint increased', 'footprint constant', 'footprint reduced']
     )
     def test_all_deltas(self, caplog, out_path, old_ram_multiplier, expect_delta_log):
         # First run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--enable-size-report'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         # Modify the older report so we can control the difference.
         # Note: if footprint tests take too long, replace first run with a prepared twister.json
         # That will increase test-to-code_under_test coupling, however.
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
                 if 'reason' not in ts:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
             f.write(json.dumps(j, indent=4))

         report_path = os.path.join(
             os.path.dirname(out_path),
             f'{os.path.basename(out_path)}.1',
             'twister.json'
         )

         # Second run
         test_platforms = ['frdm_k64f']
         path = os.path.join(TEST_DATA, 'tests', 'dummy')
         args = ['-i', '--outdir', out_path, '-T', path] + \
                ['--all-deltas'] + \
                ['--compare-report', report_path, '--show-footprint'] + \
                [val for pair in zip(
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]

         with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                 pytest.raises(SystemExit) as sys_exit:
             self.loader.exec_module(self.twister_module)

         assert str(sys_exit.value) == '0'

         assert self.FOOTPRINT_LOG in caplog.text

         if expect_delta_log:
             assert self.RAM_KEY in caplog.text
             assert self.DELTA_WARNING_RELEASE in caplog.text, \
                 'Expected footprint deltas not logged.'
         else:
             assert self.RAM_KEY not in caplog.text
             assert self.DELTA_WARNING_RELEASE not in caplog.text, \
                 'Unexpected footprint deltas logged.'
	#!/usr/bin/env python3
	# Copyright (c) 2024 Intel Corporation
	#
	# SPDX-License-Identifier: Apache-2.0
	"""
	Blackbox tests for twister's command line functions related to memory footprints.
	"""

	import importlib
	import json
	import mock
	import os
	import pytest
	import sys

	from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test
	from twisterlib.testplan import TestPlan


	@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
	class TestFootprint:
	# Log printed when entering delta calculations
	FOOTPRINT_LOG = 'running footprint_reports'

	# These warnings notify us that deltas were shown in log.
	# Coupled with the code under test.
	DELTA_WARNING_RELEASE = 'Deltas based on metrics from last release'
	DELTA_WARNING_RUN = 'Deltas based on metrics from last run'

	# Size report key we modify to control for deltas
	RAM_KEY = 'used_ram'

	@classmethod
	def setup_class(cls):
	apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
	cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
	cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
	cls.twister_module = importlib.util.module_from_spec(cls.spec)

	@classmethod
	def teardown_class(cls):
	pass

	@pytest.mark.parametrize(
	'old_ram_multiplier, expect_delta_log',
	[
	(0.75, True),
	(1.25, False),
	],
	ids=['footprint increased', 'footprint reduced']
	)
	def test_compare_report(self, caplog, out_path, old_ram_multiplier, expect_delta_log):
	# First run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--enable-size-report'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Modify the older report so we can control the difference.
	# Note: if footprint tests take too long, replace first run with a prepared twister.json
	# That will increase test-to-code_under_test coupling, however.
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	# We assume positive RAM usage.
	ts[self.RAM_KEY] *= old_ram_multiplier
	with open(os.path.join(out_path, 'twister.json'), 'w') as f:
	f.write(json.dumps(j, indent=4))

	report_path = os.path.join(
	os.path.dirname(out_path),
	f'{os.path.basename(out_path)}.1',
	'twister.json'
	)

	# Second run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--compare-report', report_path] + \
	['--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	assert self.FOOTPRINT_LOG in caplog.text

	if expect_delta_log:
	assert self.RAM_KEY in caplog.text
	assert self.DELTA_WARNING_RELEASE in caplog.text, \
	'Expected footprint deltas not logged.'
	else:
	assert self.RAM_KEY not in caplog.text
	assert self.DELTA_WARNING_RELEASE not in caplog.text, \
	'Unexpected footprint deltas logged.'

	def test_footprint_from_buildlog(self, out_path):
	# First run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	[] + \
	['--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Get values
	old_values = []
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	assert self.RAM_KEY in ts
	old_values += [ts[self.RAM_KEY]]

	# Second run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--footprint-from-buildlog'] + \
	['--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Get values
	new_values = []
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	assert self.RAM_KEY in ts
	new_values += [ts[self.RAM_KEY]]

	# There can be false positives if our calculations become too accurate.
	# Turn this test into a dummy (check only exit value) in such case.
	assert sorted(old_values) != sorted(new_values), \
	'Same values whether calculating size or using buildlog.'

	@pytest.mark.parametrize(
	'old_ram_multiplier, threshold, expect_delta_log',
	[
	(0.75, 95, False),
	(0.75, 25, True),
	],
	ids=['footprint threshold not met', 'footprint threshold met']
	)
	def test_footprint_threshold(self, caplog, out_path, old_ram_multiplier,
	threshold, expect_delta_log):
	# First run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--enable-size-report'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Modify the older report so we can control the difference.
	# Note: if footprint tests take too long, replace first run with a prepared twister.json
	# That will increase test-to-code_under_test coupling, however.
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	# We assume positive RAM usage.
	ts[self.RAM_KEY] *= old_ram_multiplier
	with open(os.path.join(out_path, 'twister.json'), 'w') as f:
	f.write(json.dumps(j, indent=4))

	report_path = os.path.join(
	os.path.dirname(out_path),
	f'{os.path.basename(out_path)}.1',
	'twister.json'
	)

	# Second run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	[f'--footprint-threshold={threshold}'] + \
	['--compare-report', report_path, '--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	assert self.FOOTPRINT_LOG in caplog.text

	if expect_delta_log:
	assert self.RAM_KEY in caplog.text
	assert self.DELTA_WARNING_RELEASE in caplog.text, \
	'Expected footprint deltas not logged.'
	else:
	assert self.RAM_KEY not in caplog.text
	assert self.DELTA_WARNING_RELEASE not in caplog.text, \
	'Unexpected footprint deltas logged.'

	@pytest.mark.parametrize(
	'flags, old_ram_multiplier, expect_delta_log',
	[
	([], 0.75, False),
	(['--show-footprint'], 0.75, True),
	],
	ids=['footprint reduced, no show', 'footprint reduced, show']
	)
	def test_show_footprint(self, caplog, out_path, flags, old_ram_multiplier, expect_delta_log):
	# First run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--enable-size-report'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Modify the older report so we can control the difference.
	# Note: if footprint tests take too long, replace first run with a prepared twister.json
	# That will increase test-to-code_under_test coupling, however.
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	# We assume positive RAM usage.
	ts[self.RAM_KEY] *= old_ram_multiplier
	with open(os.path.join(out_path, 'twister.json'), 'w') as f:
	f.write(json.dumps(j, indent=4))

	report_path = os.path.join(
	os.path.dirname(out_path),
	f'{os.path.basename(out_path)}.1',
	'twister.json'
	)

	# Second run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	flags + \
	['--compare-report', report_path] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]
	print(args)
	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	assert self.FOOTPRINT_LOG in caplog.text

	if expect_delta_log:
	assert self.RAM_KEY in caplog.text
	assert self.DELTA_WARNING_RELEASE in caplog.text, \
	'Expected footprint deltas not logged.'
	else:
	assert self.RAM_KEY not in caplog.text
	assert self.DELTA_WARNING_RELEASE not in caplog.text, \
	'Unexpected footprint deltas logged.'

	@pytest.mark.parametrize(
	'old_ram_multiplier, expect_delta_log',
	[
	(0.75, True),
	(1.25, False),
	],
	ids=['footprint increased', 'footprint reduced']
	)
	def test_last_metrics(self, caplog, out_path, old_ram_multiplier, expect_delta_log):
	# First run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--enable-size-report'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Modify the older report so we can control the difference.
	# Note: if footprint tests take too long, replace first run with a prepared twister.json
	# That will increase test-to-code_under_test coupling, however.
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	# We assume positive RAM usage.
	ts[self.RAM_KEY] *= old_ram_multiplier
	with open(os.path.join(out_path, 'twister.json'), 'w') as f:
	f.write(json.dumps(j, indent=4))

	report_path = os.path.join(
	os.path.dirname(out_path),
	f'{os.path.basename(out_path)}.1',
	'twister.json'
	)

	# Second run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--last-metrics'] + \
	['--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	assert self.FOOTPRINT_LOG in caplog.text

	if expect_delta_log:
	assert self.RAM_KEY in caplog.text
	assert self.DELTA_WARNING_RUN in caplog.text, \
	'Expected footprint deltas not logged.'
	else:
	assert self.RAM_KEY not in caplog.text
	assert self.DELTA_WARNING_RUN not in caplog.text, \
	'Unexpected footprint deltas logged.'

	second_logs = caplog.records
	caplog.clear()
	clear_log_in_test()

	# Third run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--compare-report', report_path] + \
	['--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Since second run should use the same source as the third, we should compare them.
	delta_logs = [
	record.getMessage() for record in second_logs \
	if self.RAM_KEY in record.getMessage()
	]
	assert all([log in caplog.text for log in delta_logs])

	@pytest.mark.parametrize(
	'old_ram_multiplier, expect_delta_log',
	[
	(0.75, True),
	(1.00, False),
	(1.25, True),
	],
	ids=['footprint increased', 'footprint constant', 'footprint reduced']
	)
	def test_all_deltas(self, caplog, out_path, old_ram_multiplier, expect_delta_log):
	# First run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy', 'device', 'group')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--enable-size-report'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	# Modify the older report so we can control the difference.
	# Note: if footprint tests take too long, replace first run with a prepared twister.json
	# That will increase test-to-code_under_test coupling, however.
	with open(os.path.join(out_path, 'twister.json')) as f:
	j = json.load(f)
	for ts in j['testsuites']:
	if 'reason' not in ts:
	# We assume positive RAM usage.
	ts[self.RAM_KEY] *= old_ram_multiplier
	with open(os.path.join(out_path, 'twister.json'), 'w') as f:
	f.write(json.dumps(j, indent=4))

	report_path = os.path.join(
	os.path.dirname(out_path),
	f'{os.path.basename(out_path)}.1',
	'twister.json'
	)

	# Second run
	test_platforms = ['frdm_k64f']
	path = os.path.join(TEST_DATA, 'tests', 'dummy')
	args = ['-i', '--outdir', out_path, '-T', path] + \
	['--all-deltas'] + \
	['--compare-report', report_path, '--show-footprint'] + \
	[val for pair in zip(
	['-p'] * len(test_platforms), test_platforms
	) for val in pair]

	with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
	pytest.raises(SystemExit) as sys_exit:
	self.loader.exec_module(self.twister_module)

	assert str(sys_exit.value) == '0'

	assert self.FOOTPRINT_LOG in caplog.text

	if expect_delta_log:
	assert self.RAM_KEY in caplog.text
	assert self.DELTA_WARNING_RELEASE in caplog.text, \
	'Expected footprint deltas not logged.'
	else:
	assert self.RAM_KEY not in caplog.text
	assert self.DELTA_WARNING_RELEASE not in caplog.text, \
	'Unexpected footprint deltas logged.'