| # Copyright (c) 2023 Intel Corporation. |
| # SPDX-License-Identifier: Apache-2.0 |
| |
| name: Twister BlackBox TestSuite |
| |
| on: |
| pull_request: |
| branches: |
| - main |
| - collab-* |
| - v*-branch |
| paths: |
| - 'scripts/pylib/twister/**' |
| - 'scripts/twister' |
| - 'scripts/tests/twister_blackbox/**' |
| - '.github/workflows/twister_tests_blackbox.yml' |
| |
| permissions: |
| contents: read |
| |
| env: |
| PYTHONIOENCODING: utf-8 |
| |
| jobs: |
| twister-tests: |
| name: Twister Black Box Tests |
| strategy: |
| matrix: |
| python-version: ['3.10', '3.11', '3.12', '3.13'] |
| os: [ubuntu-24.04, macos-14, windows-2022] |
| fail-fast: false |
| runs-on: ${{ matrix.os }} |
| steps: |
| - name: Checkout |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 |
| with: |
| path: zephyr |
| fetch-depth: 0 |
| |
| - name: Set Up Python ${{ matrix.python-version }} |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 |
| with: |
| python-version: ${{ matrix.python-version }} |
| cache: pip |
| cache-dependency-path: scripts/requirements-actions.txt |
| |
| - name: Setup Zephyr project |
| uses: zephyrproject-rtos/action-zephyr-setup@b2453c72966ee67b1433be22b250348d48283286 # v1.0.7 |
| with: |
| app-path: zephyr |
| toolchains: all |
| |
| - name: Run Pytest For Twister Black Box Tests |
| if: ${{ startsWith(runner.os, 'ubuntu') }} |
| working-directory: zephyr |
| shell: bash |
| env: |
| ZEPHYR_BASE: ./ |
| ZEPHYR_TOOLCHAIN_VARIANT: zephyr |
| run: | |
| export ZEPHYR_SDK_INSTALL_DIR=${{ github.workspace }}/zephyr-sdk |
| echo "Run twister tests" |
| source zephyr-env.sh |
| PYTHONPATH="./scripts/tests" pytest ./scripts/tests/twister_blackbox/ |
| |
| - name: Build firmware No. 1 - basic |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| ./scripts/twister --runtime-artifact-cleanup --force-color --inline-logs -T samples/hello_world -T samples/cpp/hello_world -v $EXTRA_TWISTER_FLAGS |
| |
| - name: Build firmware No. 2 - save and load with emulation only |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -T samples/hello_world -T samples/cpp/hello_world -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --save-tests tests.file $BASIC_FLAGS |
| ./scripts/twister --load-tests tests.file --emulation-only $BASIC_FLAGS |
| rm tests.file |
| |
| - name: Build firmware No. 3 - print out test plan |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --test-tree -T tests/kernel/spinlock $BASIC_FLAGS |
| |
| - name: Build firmware No. 4 - integration, exclude tag, filter, shuffle, dry run |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --dry-run --integration --subset 1/3 --shuffle-tests --shuffle-tests-seed 1 --filter runnable --exclude-tag audio --exclude-tag driver $BASIC_FLAGS |
| |
| - name: Build firmware No. 5 - test, arch, vendor, exclude-platform, platform-reports |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --test kernel.multiprocessing.spinlock --arch x86 --exclude-platform qemu_x86_64 --vendor qemu --platform-reports $BASIC_FLAGS |
| |
| - name: Build firmware No. 6 - subtest, platform, rom-ram report, ROM footprint report from buildlog, size report |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --sub-test kernel.multiprocessing.spinlock.minimallibc.spinlock.spinlock_basic --platform qemu_x86 --create-rom-ram-report --footprint-report ROM --enable-size-report --footprint-from-buildlog $BASIC_FLAGS |
| |
| - name: Build firmware No. 7 - list tags |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --sub-test kernel.multiprocessing.spinlock.minimallibc.spinlock.spinlock_basic --list-tags $BASIC_FLAGS |
| |
| - name: Build firmware No. 8 - list tests |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister -T tests/posix/common --list-tests $BASIC_FLAGS |
| |
| - name: Build firmware No. 9 - report flags - dir, name, suffix, summary, all-options, filtered |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --sub-test kernel.multiprocessing.spinlock.minimallibc.spinlock.spinlock_basic --platform qemu_x86 --report-dir . --report-name test_name --report-suffix suffix --report-summary 0 --report-all-options --report-filtered $BASIC_FLAGS |
| |
| - name: Build firmware No. 10 - force platform and toolchain, log level, timestamps, logfile |
| working-directory: zephyr |
| shell: bash |
| run: | |
| if [ "${{ runner.os }}" = "macOS" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --build-only" |
| elif [ "${{ runner.os }}" = "Windows" ]; then |
| EXTRA_TWISTER_FLAGS="-P native_sim --short-build-path -O /tmp/twister-out" |
| fi |
| BASIC_FLAGS="--runtime-artifact-cleanup --force-color --inline-logs -v $EXTRA_TWISTER_FLAGS" |
| ./scripts/twister --sub-test kernel.multiprocessing.spinlock.minimallibc.spinlock.spinlock_basic --force-platform --platform qemu_x86 --force-toolchain --log-level WARNING --log-file log.file $BASIC_FLAGS |
| rm log.file |
