Darwin CI: use macos-13 runners and other tweaks (#32748)
* Darwin CI: use macos-13 runners and other tweaks
According to github docs these have 4 CPUs (vs 3 for macos-12 == latest).
In practice compile steps seem to be about 7% faster.
Also use the pre-installed Python (3.12), remove the "aggregation" step, and
run crash log collection after failure (not success), and condense disk space
output to large directories only.
* CI: Save bootstrap cache right after bootrap
This avoids caching a corrupted cache in cases where a job modifies the
environment (e.g. the Darwin job hiding zap-cli). It also makes the cache
available slightly sooner.
Also add plumbing for splitting caches over buildjet and github backends.
* Remove zap-cli restore workaround
* Update darwin-tests workflow as well
diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml
index 16a2fba..749dcb7 100644
--- a/.github/actions/bootstrap/action.yaml
+++ b/.github/actions/bootstrap/action.yaml
@@ -12,49 +12,66 @@
outputs:
cache-hit:
description: "Bootstrap environment was restored from cache"
- value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs`
+ value: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit }} # dynamic action returns all outputs in `outputs`
runs:
using: "composite"
steps:
- - name: Calculate bootstrap cache key
- id: bootstrap-cache-key
+ - name: Determine bootstrap cache configuration
+ id: prepare
shell: bash
run: |
- # Calculate bootstrap cache key
+ # Determine bootstrap cache configuration
# In addition to the various setup files, the work directory matters as well,
# because the bootstrapped Pigweed environment contains absolute paths.
+ echo "Calculating bootstrap cache key for '$PWD'"
FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
- echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH"
- echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT"
+ echo key="${RUNNER_OS}-${RUNNER_ARCH}-${{ inputs.platform }}-${FINAL_HASH}" | tee -a "$GITHUB_OUTPUT"
- - uses: Wandalen/wretry.action@v1.4.10
- name: Bootstrap from cache
- id: bootstrap-cache
+ # Split caches across backends
+ case "$RUNNER_OS" in
+ macOS) echo backend=actions;;
+ *) echo backend=buildjet;;
+ esac | tee -a "$GITHUB_OUTPUT"
+
+ - name: Bootstrap from cache
+ id: restore
+ uses: ./.github/actions/dynamic
continue-on-error: true
with:
- action: buildjet/cache@v4
- attempt_limit: 3
- attempt_delay: 2000
+ action: ${{ steps.prepare.outputs.backend }}/cache/restore@v4
with: |
- key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}}
+ key: ${{ steps.prepare.outputs.key }}
path: |
- .environment
- build_overrides/pigweed_environment.gni
+ .environment
+ build_overrides/pigweed_environment.gni
- name: Run bootstrap
- if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
+ if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
env:
- PW_NO_CIPD_CACHE_DIR: Y
+ PW_NO_CIPD_CACHE_DIR: 1
+ PW_ENVSETUP_NO_BANNER: 1
shell: bash
run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
- - name: Uploading bootstrap logs
+ - name: Save bootstrap cache
+ uses: ./.github/actions/dynamic
+ if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+ continue-on-error: true
+ with:
+ action: ${{ steps.prepare.outputs.backend }}/cache/save@v4
+ with: |
+ key: ${{ steps.prepare.outputs.key }}
+ path: |
+ .environment
+ build_overrides/pigweed_environment.gni
+
+ - name: Upload bootstrap logs
uses: actions/upload-artifact@v4
- if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
+ if: ${{ always() && !env.ACT && fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
with:
name: ${{ inputs.bootstrap-log-name }}
path: |
- .environment/gn_out/.ninja_log
- .environment/pigweed-venv/*.log
+ .environment/gn_out/.ninja_log
+ .environment/pigweed-venv/*.log
diff --git a/.github/actions/dynamic/action.yaml b/.github/actions/dynamic/action.yaml
new file mode 100644
index 0000000..7cee233
--- /dev/null
+++ b/.github/actions/dynamic/action.yaml
@@ -0,0 +1,38 @@
+name: Dynamic
+description: Dynamically resolves another GitHub action
+inputs:
+ action:
+ description: Action reference
+ required: true
+ with:
+ description: Action inputs as multi-line YAML string
+ required: false
+ default: ""
+outputs:
+ outputs:
+ description: JSON object of outputs from the action
+ value: ${{ steps.run.outputs.outputs }}
+runs:
+ using: composite
+ steps:
+ - name: Instantiate
+ shell: bash
+ run: |
+ # Dynamically invoke ${{ inputs.action }}
+ with='${{ inputs.with }}'
+ [[ -z "$with" ]] || with="$(echo ' with:'; sed -e 's/^/ /' <<<"$with")"
+ mkdir -p ./.tmp/dynamic-action-instance
+ cat <<END >./.tmp/dynamic-action-instance/action.yaml
+ runs:
+ using: composite
+ steps:
+ - id: run
+ uses: ${{ inputs.action }}
+ $with
+ outputs:
+ outputs:
+ value: $(echo '$'){{ toJSON(steps.run.outputs) }}
+ END
+ - name: Run
+ id: run
+ uses: ./.tmp/dynamic-action-instance
diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml
index e6e14ec..56fd585 100644
--- a/.github/workflows/darwin-tests.yaml
+++ b/.github/workflows/darwin-tests.yaml
@@ -49,7 +49,7 @@
LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
if: github.actor != 'restyled-io[bot]'
- runs-on: macos-latest
+ runs-on: macos-13
steps:
- name: Checkout
diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml
index bacfb90..6e12f1d 100644
--- a/.github/workflows/darwin.yaml
+++ b/.github/workflows/darwin.yaml
@@ -31,7 +31,7 @@
framework:
name: Build framework
if: github.actor != 'restyled-io[bot]'
- runs-on: macos-latest
+ runs-on: macos-13
strategy:
matrix:
options: # We don't need a full matrix
@@ -48,30 +48,28 @@
steps:
- name: Checkout
uses: actions/checkout@v4
- - name: Setup Environment
- run: brew install python@3.9
- name: Checkout submodules & Bootstrap
uses: ./.github/actions/checkout-submodules-and-bootstrap
with:
platform: darwin
bootstrap-log-name: bootstrap-logs-framework-${{ matrix.options.flavor }}
- name: Block zap-cli from being used
+ env:
+ PW_ENVSETUP_NO_BANNER: 1
run: |
# Framework builds are NOT expected to require zap-cli
- scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli)) && mv $D/zap-cli $D/zap-cli.moved'
+ scripts/run_in_build_env.sh 'rm -- "$(which zap-cli)"'
# run_in_build_env.sh is used to ensure PATH is set to something that would otherwise find zap-cli
scripts/run_in_build_env.sh '(zap-cli --version && exit 1) || exit 0'
- name: Build
working-directory: src/darwin/Framework
run: xcodebuild -target "Matter" ${{ matrix.options.arguments }}
- # Now restore zap-cli, so that bootstrap caching does not cache the state when it's been renamed.
- - name: Make zap-cli work again
- run: scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli.moved)) && mv $D/zap-cli.moved $D/zap-cli'
tests:
name: Run framework tests
if: github.actor != 'restyled-io[bot]'
- runs-on: macos-latest
+ needs: [ framework ] # serialize to avoid running to many parallel macos runners
+ runs-on: macos-13
strategy:
matrix:
options: # We don't need a full matrix
@@ -85,8 +83,6 @@
steps:
- name: Checkout
uses: actions/checkout@v4
- - name: Setup Environment
- run: brew install python@3.9
- name: Checkout submodules & Bootstrap
uses: ./.github/actions/checkout-submodules-and-bootstrap
with:
@@ -120,6 +116,7 @@
OTHER_CFLAGS='${inherited} -Werror -Wconversion' CHIP_IS_BLE=NO GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1 ${{ matrix.options.defines }}' \
> >(tee /tmp/darwin/framework-tests/darwin-tests.log) 2> >(tee /tmp/darwin/framework-tests/darwin-tests-err.log >&2)
- name: Collect crash logs
+ if: failure() && !env.ACT
run: |
mkdir -p /tmp/darwin/framework-tests
find ~/Library/Developer/Xcode/DerivedData /Library/Logs/DiagnosticReports -name '*.ips' -print0 | xargs -0 -J % cp % /tmp/darwin/framework-tests
@@ -134,12 +131,11 @@
tv-casting-bridge:
name: Build TV Casting Bridge example
if: github.actor != 'restyled-io[bot]'
- runs-on: macos-latest
+ needs: [ framework ] # serialize to avoid running to many parallel macos runners
+ runs-on: macos-13
steps:
- name: Checkout
uses: actions/checkout@v4
- - name: Setup Environment
- run: brew install python@3.9
- name: Checkout submodules & Bootstrap
uses: ./.github/actions/checkout-submodules-and-bootstrap
with:
@@ -147,11 +143,3 @@
- name: Build
working-directory: examples/tv-casting-app/darwin/MatterTvCastingBridge
run: xcodebuild -target "MatterTvCastingBridge" -sdk iphoneos
-
- darwin:
- name: Build Darwin # Matches the previous monolithic build that's marked "required" for PRs
- needs: [ framework, tests ]
- runs-on: macos-latest
- steps:
- - name: Done
- run: 'true' # nothing to do
diff --git a/scripts/dump_diskspace_info.sh b/scripts/dump_diskspace_info.sh
index 2e947f4..177e1ba 100755
--- a/scripts/dump_diskspace_info.sh
+++ b/scripts/dump_diskspace_info.sh
@@ -36,7 +36,7 @@
echo "Storage Space Used By Key Directories:"
for directory in "${listOfDirectories[@]}"; do
if [ -d "$directory" ]; then
- du -d1 -h "$directory" | sort -h
+ du -d1 -h "$directory" | grep '^[[:space:]]*[0-9.]*[MG][[:space:]]' | sort -rh
echo
fi
done
