Skip to content

Commit d26d2e5

Browse files
ksperling-appleksperling-apple
authored
Include the work directory path in the bootstrap cache key (to account for containerized/non-containerized builds) (project-chip#32638)
* Silabs requirements are for non-Docker builds * CI: Don't run bootstrap if we restored from cache Handle caching and log upload directly within the bootstrap action, and don't perform the actual bootstrap if we have a cache hit. Include the platform name in the cache key. * Include the work directory path in the bootstrap cache key This is because the bootstrapped Pigweed environment does not like to be relocated. Also don't touch submodules in bootstrap, other than ensuring that Pigweed is available. * Include runner.arch as well We may start using macos-14 runners in the future which are arm.
1 parent 1e4c663 commit d26d2e5

File tree

9 files changed

+67
-66
lines changed

9 files changed

+67
-66
lines changed

‎.github/actions/bootstrap-cache/action.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: Bootstrap cache
2-
description: Bootstrap cache
2+
description: Bootstrap cache (deprecated)
33
runs:
44
using: "composite"
55
steps:

‎.github/actions/bootstrap/action.yaml

+49-2
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,56 @@ inputs:
55
description: "Platform name"
66
required: false
77
default: none
8+
bootstrap-log-name:
9+
description: "Bootstrap log name"
10+
required: false
11+
default: bootstrap-logs-${{ github.job }}
12+
outputs:
13+
cache-hit:
14+
description: "Bootstrap environment was restored from cache"
15+
value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs`
16+
817
runs:
918
using: "composite"
1019
steps:
11-
- name: Bootstrap
20+
- name: Calculate bootstrap cache key
21+
id: bootstrap-cache-key
22+
shell: bash
23+
run: |
24+
# Calculate bootstrap cache key
25+
# In addition to the various setup files, the work directory matters as well,
26+
# because the bootstrapped Pigweed environment contains absolute paths.
27+
FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
28+
FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
29+
echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH"
30+
echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT"
31+
32+
- uses: Wandalen/wretry.action@v1.4.10
33+
name: Bootstrap from cache
34+
id: bootstrap-cache
35+
continue-on-error: true
36+
with:
37+
action: buildjet/cache@v4
38+
attempt_limit: 3
39+
attempt_delay: 2000
40+
with: |
41+
key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}}
42+
path: |
43+
.environment
44+
build_overrides/pigweed_environment.gni
45+
46+
- name: Run bootstrap
47+
if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
48+
env:
49+
PW_NO_CIPD_CACHE_DIR: Y
1250
shell: bash
13-
run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }}
51+
run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
52+
53+
- name: Uploading bootstrap logs
54+
uses: actions/upload-artifact@v4
55+
if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
56+
with:
57+
name: ${{ inputs.bootstrap-log-name }}
58+
path: |
59+
.environment/gn_out/.ninja_log
60+
.environment/pigweed-venv/*.log

‎.github/actions/checkout-submodules-and-bootstrap/action.yaml

+1-8
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,14 @@ runs:
2626
with:
2727
platform: ${{ inputs.platform }}
2828
extra-parameters: ${{ inputs.extra-submodule-parameters }}
29-
- name: Bootstrap Cache
30-
uses: ./.github/actions/bootstrap-cache
3129
- name: Bootstrap
3230
uses: ./.github/actions/bootstrap
33-
env:
34-
PW_NO_CIPD_CACHE_DIR: Y
3531
with:
3632
platform: ${{ inputs.platform }}
33+
bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
3734
- name: Dump disk info after checkout submodule & Bootstrap
3835
shell: bash
3936
run: scripts/dump_diskspace_info.sh
40-
- name: Upload Bootstrap Logs
41-
uses: ./.github/actions/upload-bootstrap-logs
42-
with:
43-
bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
4437
- name: Work around TSAN ASLR issues
4538
if: runner.os == 'Linux' && !env.ACT
4639
shell: bash

‎.github/actions/upload-bootstrap-logs/action.yaml

-18
This file was deleted.

‎.github/workflows/cirque.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ jobs:
5757
with:
5858
platform: linux
5959

60+
# TODO: Is what's being cached here actually compatible with a regular bootstrap?
6061
- name: Bootstrap Cache
6162
uses: ./.github/actions/bootstrap-cache
6263
- name: Bootstrap Cirque

‎.github/workflows/fuzzing-build.yaml

-8
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,8 @@ jobs:
4646
run: |
4747
mkdir objdir-clone || true
4848
49-
- name: Bootstrap Cache
50-
uses: ./.github/actions/bootstrap-cache
5149
- name: Bootstrap
5250
uses: ./.github/actions/bootstrap
53-
- name: Upload Bootstrap Logs
54-
uses: ./.github/actions/upload-bootstrap-logs
5551

5652
- name: Build all-clusters-app
5753
run: |
@@ -84,12 +80,8 @@ jobs:
8480
run: |
8581
mkdir objdir-clone || true
8682
87-
- name: Bootstrap Cache
88-
uses: ./.github/actions/bootstrap-cache
8983
- name: Bootstrap
9084
uses: ./.github/actions/bootstrap
91-
- name: Upload Bootstrap Logs
92-
uses: ./.github/actions/upload-bootstrap-logs
9385

9486
- name: Build all-clusters-app
9587
run: |

‎.github/workflows/release_artifacts.yaml

-8
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,8 @@ jobs:
3939
uses: actions/checkout@v4
4040
with:
4141
ref: "${{ github.event.inputs.releaseTag }}"
42-
- name: Bootstrap Cache
43-
uses: ./.github/actions/bootstrap-cache
4442
- name: Bootstrap
4543
uses: ./.github/actions/bootstrap
46-
- name: Upload Bootstrap Logs
47-
uses: ./.github/actions/upload-bootstrap-logs
4844

4945
- name: Build
5046
run: scripts/examples/esp_example.sh all-clusters-app
@@ -74,12 +70,8 @@ jobs:
7470
uses: actions/checkout@v4
7571
with:
7672
ref: "${{ github.event.inputs.releaseTag }}"
77-
- name: Bootstrap Cache
78-
uses: ./.github/actions/bootstrap-cache
7973
- name: Bootstrap
8074
uses: ./.github/actions/bootstrap
81-
- name: Upload Bootstrap Logs
82-
uses: ./.github/actions/upload-bootstrap-logs
8375

8476
- name: Build example EFR32 Lock App
8577
run: scripts/examples/gn_silabs_example.sh examples/lock-app/efr32/

‎scripts/setup/bootstrap.sh

+15-21
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ _install_additional_pip_requirements() {
2121
# figure out additional pip install items
2222
while [ $# -gt 0 ]; do
2323
case $1 in
24-
-p | --platform)
25-
_SETUP_PLATFORM=$2
26-
shift # argument
27-
shift # value
28-
;;
29-
*)
30-
shift
31-
;;
24+
-p | --platform)
25+
_SETUP_PLATFORM=$2
26+
shift # argument
27+
shift # value
28+
;;
29+
*)
30+
shift
31+
;;
3232
esac
3333
done
3434

@@ -41,7 +41,7 @@ _install_additional_pip_requirements() {
4141

4242
for platform in ${_SETUP_PLATFORM}; do
4343
# Allow none as an alias of nothing extra installed (like -p none)
44-
if [ "$platform" != "none" ]; then
44+
if [ "$platform" != "none" -a -e "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" ]; then
4545
echo "Installing pip requirements for $platform..."
4646
pip install -q \
4747
-r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \
@@ -66,7 +66,7 @@ _bootstrap_or_activate() {
6666
local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}"
6767
local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}"
6868
# Strip off the 'scripts[/setup]' directory, leaving the root of the repo.
69-
_CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." > /dev/null && pwd)"
69+
_CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." >/dev/null && pwd)"
7070

7171
local _CONFIG_FILE="scripts/setup/environment.json"
7272

@@ -75,16 +75,6 @@ _bootstrap_or_activate() {
7575
unset PW_CONFIG_FILE
7676
fi
7777

78-
if [ ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
79-
# Make sure our submodule remotes are correct for this revision.
80-
git submodule sync --recursive
81-
git submodule update --init
82-
elif [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" ]; then
83-
# In this case, only update already checked out submodules.
84-
git submodule sync --recursive
85-
git submodule update
86-
fi
87-
8878
PW_BRANDING_BANNER="$_CHIP_ROOT/scripts/setup/banner.txt"
8979
export PW_BRANDING_BANNER
9080

@@ -94,6 +84,10 @@ _bootstrap_or_activate() {
9484
PW_ROOT="$_CHIP_ROOT/third_party/pigweed/repo"
9585
export PW_ROOT
9686

87+
# Update or init the pigweed submodule if necessary. Don't touch any other submodules.
88+
if [ "$_BOOTSTRAP_NAME" = "bootstrap.sh" -a ! -f "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh" ]; then
89+
git submodule update --init "$_CHIP_ROOT/third_party/pigweed/repo"
90+
fi
9791
. "$_CHIP_ROOT/third_party/pigweed/repo/pw_env_setup/util.sh"
9892

9993
_chip_bootstrap_banner() {
@@ -105,7 +99,7 @@ _bootstrap_or_activate() {
10599

106100
local _PW_BANNER_FUNC="_chip_bootstrap_banner"
107101

108-
# Force the Pigweed environment directory to be '.environment'
102+
# Default the Pigweed environment directory to be '.environment'
109103
if [ -z "$PW_ENVIRONMENT_ROOT" ]; then
110104
export PW_ENVIRONMENT_ROOT="$PW_PROJECT_ROOT/.environment"
111105
fi

‎scripts/setup/requirements.silabs_docker.txt ‎scripts/setup/requirements.silabs.txt

File renamed without changes.

0 commit comments

Comments
 (0)