YAML linter: Add check for manual steps, remove known bad unit tests (#35493)
* YAML linter: Add check for manual steps, remove known bad unit tests
* test bad change
* Restyled by isort
* Trying again for CI - checkout had no depth
* testing - I can't repro locally
* ha...removed my test bad test
* more testing...ci is different than local and act
* debugging in ci...fun times
* Restyled by whitespace
* let's just run the one, ya?
* Fix git on the VM
* Actually error though
* Revert "let's just run the one, ya?"
This reverts commit e3c045dd01c65300d04d8d12cba97c4a24bcdb62.
* Remove the added bad step
---------
Co-authored-by: Restyled.io <commits@restyled.io>
diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml
index 44d545a..3f8f54d 100644
--- a/.github/workflows/cert_test_checks.yaml
+++ b/.github/workflows/cert_test_checks.yaml
@@ -18,6 +18,8 @@
pull_request:
paths:
- "src/app/tests/suites/certification/**"
+permissions:
+ contents: read
jobs:
check-certification-tests:
@@ -30,6 +32,8 @@
steps:
- name: Checkout
uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
- name: Run checks
run: |
python3 scripts/tests/matter_yaml_linter.py
diff --git a/scripts/tests/matter_yaml_linter.py b/scripts/tests/matter_yaml_linter.py
index 4fdcb76..308e151 100644
--- a/scripts/tests/matter_yaml_linter.py
+++ b/scripts/tests/matter_yaml_linter.py
@@ -15,6 +15,7 @@
# limitations under the License.
import os
import re
+import subprocess
import sys
from pathlib import Path
@@ -23,15 +24,12 @@
DEFAULT_CHIP_ROOT = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..'))
-# TODO: These tests need to be re-written. Please see https://github.com/project-chip/connectedhomeip/issues/32620
-KNOWN_BAD_UNIT_TESTING = set(('Test_TC_S_2_2.yaml', 'Test_TC_S_2_3.yaml'))
-
def _is_cert_test(path):
return "certification" in os.path.dirname(path)
-def main():
+def check_unit_testing():
bad_tests = set()
for test in AllChipToolYamlTests(use_short_run_name=False):
with open(test.run_name, "r") as f:
@@ -47,10 +45,37 @@
print(f'\t{line+1}: {val}')
bad_tests.add(Path(test.run_name).name)
- if bad_tests - KNOWN_BAD_UNIT_TESTING:
+ if bad_tests:
return 1
return 0
+def check_manual_steps():
+ # Doing this on a test-by-test basis so the log message is more obvious
+ bad_test = False
+ # We are operating in a VM, and although there is a checkout, it is working in a scratch directory
+ # where the ownership is different than the runner.
+ # Adding an exception for this directory so that git can function properly.
+ subprocess.run("git config --global --add safe.directory '*'", shell=True)
+ for test in AllChipToolYamlTests(use_short_run_name=False):
+
+ cmd = f'git diff HEAD^..HEAD --unified=0 -- {test.run_name}'
+ output = subprocess.check_output(cmd, shell=True).decode().splitlines()
+ user_prompt_added = [line for line in output if re.search(r'^\+.*UserPrompt.*', line)]
+ user_prompt_removed = [line for line in output if re.search(r'^\-.*UserPrompt.*', line)]
+ if len(user_prompt_added) > len(user_prompt_removed):
+ print(f'Found YAML test with additional manual steps: {test.name}')
+ bad_test = True
+ if bad_test:
+ return 1
+ return 0
+
+
+def main():
+ ret = check_unit_testing()
+ ret += check_manual_steps()
+ return ret
+
+
if __name__ == '__main__':
sys.exit(main())
