Include the work directory path in the bootstrap cache key (to account for containerized/non-containerized builds) (#32638)
* Silabs requirements are for non-Docker builds
* CI: Don't run bootstrap if we restored from cache
Handle caching and log upload directly within the bootstrap action, and don't
perform the actual bootstrap if we have a cache hit. Include the platform name
in the cache key.
* Include the work directory path in the bootstrap cache key
This is because the bootstrapped Pigweed environment does not like to be
relocated. Also don't touch submodules in bootstrap, other than ensuring that
Pigweed is available.
* Include runner.arch as well
We may start using macos-14 runners in the future which are arm.
diff --git a/.github/actions/bootstrap-cache/action.yaml b/.github/actions/bootstrap-cache/action.yaml
index b22ed1e..9a883ec 100644
--- a/.github/actions/bootstrap-cache/action.yaml
+++ b/.github/actions/bootstrap-cache/action.yaml
@@ -1,5 +1,5 @@
name: Bootstrap cache
-description: Bootstrap cache
+description: Bootstrap cache (deprecated)
runs:
using: "composite"
steps:
diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml
index 8f94830..16a2fba 100644
--- a/.github/actions/bootstrap/action.yaml
+++ b/.github/actions/bootstrap/action.yaml
@@ -5,9 +5,56 @@
description: "Platform name"
required: false
default: none
+ bootstrap-log-name:
+ description: "Bootstrap log name"
+ required: false
+ default: bootstrap-logs-${{ github.job }}
+outputs:
+ cache-hit:
+ description: "Bootstrap environment was restored from cache"
+ value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs`
+
runs:
using: "composite"
steps:
- - name: Bootstrap
+ - name: Calculate bootstrap cache key
+ id: bootstrap-cache-key
shell: bash
- run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }}
+ run: |
+ # Calculate bootstrap cache key
+ # In addition to the various setup files, the work directory matters as well,
+ # because the bootstrapped Pigweed environment contains absolute paths.
+ FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
+ FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
+ echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH"
+ echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT"
+
+ - uses: Wandalen/wretry.action@v1.4.10
+ name: Bootstrap from cache
+ id: bootstrap-cache
+ continue-on-error: true
+ with:
+ action: buildjet/cache@v4
+ attempt_limit: 3
+ attempt_delay: 2000
+ with: |
+ key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}}
+ path: |
+ .environment
+ build_overrides/pigweed_environment.gni
+
+ - name: Run bootstrap
+ if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
+ env:
+ PW_NO_CIPD_CACHE_DIR: Y
+ shell: bash
+ run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
+
+ - name: Uploading bootstrap logs
+ uses: actions/upload-artifact@v4
+ if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
+ with:
+ name: ${{ inputs.bootstrap-log-name }}
+ path: |
+ .environment/gn_out/.ninja_log
+ .environment/pigweed-venv/*.log
diff --git a/.github/actions/checkout-submodules-and-bootstrap/action.yaml b/.github/actions/checkout-submodules-and-bootstrap/action.yaml
index 7424ca5..df3fdff 100644
--- a/.github/actions/checkout-submodules-and-bootstrap/action.yaml
+++ b/.github/actions/checkout-submodules-and-bootstrap/action.yaml
@@ -26,21 +26,14 @@
with:
platform: ${{ inputs.platform }}
extra-parameters: ${{ inputs.extra-submodule-parameters }}
- - name: Bootstrap Cache
- uses: ./.github/actions/bootstrap-cache
- name: Bootstrap
uses: ./.github/actions/bootstrap
- env:
- PW_NO_CIPD_CACHE_DIR: Y
with:
platform: ${{ inputs.platform }}
+ bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
- name: Dump disk info after checkout submodule & Bootstrap
shell: bash
run: scripts/dump_diskspace_info.sh
- - name: Upload Bootstrap Logs
- uses: ./.github/actions/upload-bootstrap-logs
- with:
- bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
- name: Work around TSAN ASLR issues
if: runner.os == 'Linux' && !env.ACT
shell: bash
diff --git a/.github/actions/upload-bootstrap-logs/action.yaml b/.github/actions/upload-bootstrap-logs/action.yaml
deleted file mode 100644
index 522058c..0000000
--- a/.github/actions/upload-bootstrap-logs/action.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: Upload bootstrap logs
-description: Upload bootstrap logs
-inputs:
- bootstrap-log-name:
- description: "Bootstrap log name"
- required: false
- default: bootstrap-logs-${{ github.job }}
-runs:
- using: "composite"
- steps:
- - name: Uploading bootstrap logs
- uses: actions/upload-artifact@v4
- if: ${{ always() && !env.ACT }}
- with:
- name: ${{ inputs.bootstrap-log-name }}
- path: |
- .environment/gn_out/.ninja_log
- .environment/pigweed-venv/*.log
diff --git a/.github/workflows/cirque.yaml b/.github/workflows/cirque.yaml
index 1f0b7c5..919085a 100644
--- a/.github/workflows/cirque.yaml
+++ b/.github/workflows/cirque.yaml
@@ -57,6 +57,7 @@
with:
platform: linux
+ # TODO: Is what's being cached here actually compatible with a regular bootstrap?
- name: Bootstrap Cache
uses: ./.github/actions/bootstrap-cache
- name: Bootstrap Cirque
diff --git a/.github/workflows/fuzzing-build.yaml b/.github/workflows/fuzzing-build.yaml
index ee19873..f354ce3 100644
--- a/.github/workflows/fuzzing-build.yaml
+++ b/.github/workflows/fuzzing-build.yaml
@@ -46,12 +46,8 @@
run: |
mkdir objdir-clone || true
- - name: Bootstrap Cache
- uses: ./.github/actions/bootstrap-cache
- name: Bootstrap
uses: ./.github/actions/bootstrap
- - name: Upload Bootstrap Logs
- uses: ./.github/actions/upload-bootstrap-logs
- name: Build all-clusters-app
run: |
@@ -84,12 +80,8 @@
run: |
mkdir objdir-clone || true
- - name: Bootstrap Cache
- uses: ./.github/actions/bootstrap-cache
- name: Bootstrap
uses: ./.github/actions/bootstrap
- - name: Upload Bootstrap Logs
- uses: ./.github/actions/upload-bootstrap-logs
- name: Build all-clusters-app
run: |
diff --git a/.github/workflows/release_artifacts.yaml b/.github/workflows/release_artifacts.yaml
index 5aa38ec..57a1ce7 100644
--- a/.github/workflows/release_artifacts.yaml
+++ b/.github/workflows/release_artifacts.yaml
@@ -39,12 +39,8 @@
uses: actions/checkout@v4
with:
ref: "${{ github.event.inputs.releaseTag }}"
- - name: Bootstrap Cache
- uses: ./.github/actions/bootstrap-cache
- name: Bootstrap
uses: ./.github/actions/bootstrap
- - name: Upload Bootstrap Logs
- uses: ./.github/actions/upload-bootstrap-logs
- name: Build
run: scripts/examples/esp_example.sh all-clusters-app
@@ -74,12 +70,8 @@
uses: actions/checkout@v4
with:
ref: "${{ github.event.inputs.releaseTag }}"
- - name: Bootstrap Cache
- uses: ./.github/actions/bootstrap-cache
- name: Bootstrap
uses: ./.github/actions/bootstrap
- - name: Upload Bootstrap Logs
- uses: ./.github/actions/upload-bootstrap-logs
- name: Build example EFR32 Lock App
run: scripts/examples/gn_silabs_example.sh examples/lock-app/efr32/
diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh
index 1b81321..0a2a898 100644
--- a/scripts/setup/bootstrap.sh
+++ b/scripts/setup/bootstrap.sh
@@ -21,14 +21,14 @@
# figure out additional pip install items
while [ $# -gt 0 ]; do
case $1 in
- -p | --platform)
- _SETUP_PLATFORM=$2
- shift # argument
- shift # value
- ;;
- *)
- shift
- ;;
+ -p | --platform)
+ _SETUP_PLATFORM=$2
+ shift # argument
+ shift # value
+ ;;
+ *)
+ shift
+ ;;
esac
done
@@ -41,7 +41,7 @@
for platform in ${_SETUP_PLATFORM}; do
# Allow none as an alias of nothing extra installed (like -p none)
- if [ "$platform" != "none" ]; then
+ if [ "$platform" != "none" -a -e "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" ]; then
echo "Installing pip requirements for $platform..."
pip install -q \
-r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \
@@ -66,7 +66,7 @@
local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}"
local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}"
# Strip off the 'scripts[/setup]' directory, leaving the root of the repo.
- _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." > /dev/null && pwd)"
+ _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." >/dev/null && pwd)"
local _CONFIG_FILE="scripts/setup/environment.json"
@@ -75,16 +75,6 @@
unset PW_CONFIG_FILE
fi
- if [ ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
- # Make sure our submodule remotes are correct for this revision.
- git submodule sync --recursive
- git submodule update --init
- elif [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ]; then
- # In this case, only update already checked out submodules.
- git submodule sync --recursive
- git submodule update
- fi
-
PW_BRANDING_BANNER="$_CHIP_ROOT/scripts/setup/banner.txt"
export PW_BRANDING_BANNER
@@ -94,6 +84,10 @@
PW_ROOT="$_CHIP_ROOT/third_party/pigweed/repo"
export PW_ROOT
+ # Update or init the pigweed submodule if necessary. Don't touch any other submodules.
+ if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" -a ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
+ git submodule update --init "$_CHIP_ROOT/third_party/pigweed/repo"
+ fi
. "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh"
_chip_bootstrap_banner() {
@@ -105,7 +99,7 @@
local _PW_BANNER_FUNC="_chip_bootstrap_banner"
- # Force the Pigweed environment directory to be '.environment'
+ # Default the Pigweed environment directory to be '.environment'
if [ -z "$PW_ENVIRONMENT_ROOT" ]; then
export PW_ENVIRONMENT_ROOT="$PW_PROJECT_ROOT/.environment"
fi
diff --git a/scripts/setup/requirements.silabs_docker.txt b/scripts/setup/requirements.silabs.txt
similarity index 100%
rename from scripts/setup/requirements.silabs_docker.txt
rename to scripts/setup/requirements.silabs.txt
