Skip to content

Commit 036e7bf

Browse files
authoredMar 28, 2024
Darwin CI: use macos-13 runners and other tweaks (#32748)
* Darwin CI: use macos-13 runners and other tweaks According to github docs these have 4 CPUs (vs 3 for macos-12 == latest). In practice compile steps seem to be about 7% faster. Also use the pre-installed Python (3.12), remove the "aggregation" step, and run crash log collection after failure (not success), and condense disk space output to large directories only. * CI: Save bootstrap cache right after bootrap This avoids caching a corrupted cache in cases where a job modifies the environment (e.g. the Darwin job hiding zap-cli). It also makes the cache available slightly sooner. Also add plumbing for splitting caches over buildjet and github backends. * Remove zap-cli restore workaround * Update darwin-tests workflow as well
1 parent 456e785 commit 036e7bf

File tree

5 files changed

+87
-44
lines changed

5 files changed

+87
-44
lines changed
 

‎.github/actions/bootstrap/action.yaml

+38-21
Original file line numberDiff line numberDiff line change
@@ -12,49 +12,66 @@ inputs:
1212
outputs:
1313
cache-hit:
1414
description: "Bootstrap environment was restored from cache"
15-
value: ${{ fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit }} # retry returns all outputs in `outputs`
15+
value: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit }} # dynamic action returns all outputs in `outputs`
1616

1717
runs:
1818
using: "composite"
1919
steps:
20-
- name: Calculate bootstrap cache key
21-
id: bootstrap-cache-key
20+
- name: Determine bootstrap cache configuration
21+
id: prepare
2222
shell: bash
2323
run: |
24-
# Calculate bootstrap cache key
24+
# Determine bootstrap cache configuration
2525
# In addition to the various setup files, the work directory matters as well,
2626
# because the bootstrapped Pigweed environment contains absolute paths.
27+
echo "Calculating bootstrap cache key for '$PWD'"
2728
FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
2829
FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
29-
echo "Calculated bootstrap cache key for '$PWD': $FINAL_HASH"
30-
echo "hash=$FINAL_HASH" >> "$GITHUB_OUTPUT"
30+
echo key="${RUNNER_OS}-${RUNNER_ARCH}-${{ inputs.platform }}-${FINAL_HASH}" | tee -a "$GITHUB_OUTPUT"
3131
32-
- uses: Wandalen/wretry.action@v1.4.10
33-
name: Bootstrap from cache
34-
id: bootstrap-cache
32+
# Split caches across backends
33+
case "$RUNNER_OS" in
34+
macOS) echo backend=actions;;
35+
*) echo backend=buildjet;;
36+
esac | tee -a "$GITHUB_OUTPUT"
37+
38+
- name: Bootstrap from cache
39+
id: restore
40+
uses: ./.github/actions/dynamic
3541
continue-on-error: true
3642
with:
37-
action: buildjet/cache@v4
38-
attempt_limit: 3
39-
attempt_delay: 2000
43+
action: ${{ steps.prepare.outputs.backend }}/cache/restore@v4
4044
with: |
41-
key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.platform }}-${{ steps.bootstrap-cache-key.outputs.hash}}
45+
key: ${{ steps.prepare.outputs.key }}
4246
path: |
43-
.environment
44-
build_overrides/pigweed_environment.gni
47+
.environment
48+
build_overrides/pigweed_environment.gni
4549
4650
- name: Run bootstrap
47-
if: fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
51+
if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
4852
env:
49-
PW_NO_CIPD_CACHE_DIR: Y
53+
PW_NO_CIPD_CACHE_DIR: 1
54+
PW_ENVSETUP_NO_BANNER: 1
5055
shell: bash
5156
run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
5257

53-
- name: Uploading bootstrap logs
58+
- name: Save bootstrap cache
59+
uses: ./.github/actions/dynamic
60+
if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
61+
continue-on-error: true
62+
with:
63+
action: ${{ steps.prepare.outputs.backend }}/cache/save@v4
64+
with: |
65+
key: ${{ steps.prepare.outputs.key }}
66+
path: |
67+
.environment
68+
build_overrides/pigweed_environment.gni
69+
70+
- name: Upload bootstrap logs
5471
uses: actions/upload-artifact@v4
55-
if: always() && !env.ACT && fromJSON(steps.bootstrap-cache.outputs.outputs).cache-hit != 'true'
72+
if: ${{ always() && !env.ACT && fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
5673
with:
5774
name: ${{ inputs.bootstrap-log-name }}
5875
path: |
59-
.environment/gn_out/.ninja_log
60-
.environment/pigweed-venv/*.log
76+
.environment/gn_out/.ninja_log
77+
.environment/pigweed-venv/*.log

‎.github/actions/dynamic/action.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Dynamic
2+
description: Dynamically resolves another GitHub action
3+
inputs:
4+
action:
5+
description: Action reference
6+
required: true
7+
with:
8+
description: Action inputs as multi-line YAML string
9+
required: false
10+
default: ""
11+
outputs:
12+
outputs:
13+
description: JSON object of outputs from the action
14+
value: ${{ steps.run.outputs.outputs }}
15+
runs:
16+
using: composite
17+
steps:
18+
- name: Instantiate
19+
shell: bash
20+
run: |
21+
# Dynamically invoke ${{ inputs.action }}
22+
with='${{ inputs.with }}'
23+
[[ -z "$with" ]] || with="$(echo ' with:'; sed -e 's/^/ /' <<<"$with")"
24+
mkdir -p ./.tmp/dynamic-action-instance
25+
cat <<END >./.tmp/dynamic-action-instance/action.yaml
26+
runs:
27+
using: composite
28+
steps:
29+
- id: run
30+
uses: ${{ inputs.action }}
31+
$with
32+
outputs:
33+
outputs:
34+
value: $(echo '$'){{ toJSON(steps.run.outputs) }}
35+
END
36+
- name: Run
37+
id: run
38+
uses: ./.tmp/dynamic-action-instance

‎.github/workflows/darwin-tests.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
5050

5151
if: github.actor != 'restyled-io[bot]'
52-
runs-on: macos-latest
52+
runs-on: macos-13
5353

5454
steps:
5555
- name: Checkout

‎.github/workflows/darwin.yaml

+9-21
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
framework:
3232
name: Build framework
3333
if: github.actor != 'restyled-io[bot]'
34-
runs-on: macos-latest
34+
runs-on: macos-13
3535
strategy:
3636
matrix:
3737
options: # We don't need a full matrix
@@ -48,30 +48,28 @@ jobs:
4848
steps:
4949
- name: Checkout
5050
uses: actions/checkout@v4
51-
- name: Setup Environment
52-
run: brew install python@3.9
5351
- name: Checkout submodules & Bootstrap
5452
uses: ./.github/actions/checkout-submodules-and-bootstrap
5553
with:
5654
platform: darwin
5755
bootstrap-log-name: bootstrap-logs-framework-${{ matrix.options.flavor }}
5856
- name: Block zap-cli from being used
57+
env:
58+
PW_ENVSETUP_NO_BANNER: 1
5959
run: |
6060
# Framework builds are NOT expected to require zap-cli
61-
scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli)) && mv $D/zap-cli $D/zap-cli.moved'
61+
scripts/run_in_build_env.sh 'rm -- "$(which zap-cli)"'
6262
# run_in_build_env.sh is used to ensure PATH is set to something that would otherwise find zap-cli
6363
scripts/run_in_build_env.sh '(zap-cli --version && exit 1) || exit 0'
6464
- name: Build
6565
working-directory: src/darwin/Framework
6666
run: xcodebuild -target "Matter" ${{ matrix.options.arguments }}
67-
# Now restore zap-cli, so that bootstrap caching does not cache the state when it's been renamed.
68-
- name: Make zap-cli work again
69-
run: scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli.moved)) && mv $D/zap-cli.moved $D/zap-cli'
7067

7168
tests:
7269
name: Run framework tests
7370
if: github.actor != 'restyled-io[bot]'
74-
runs-on: macos-latest
71+
needs: [ framework ] # serialize to avoid running to many parallel macos runners
72+
runs-on: macos-13
7573
strategy:
7674
matrix:
7775
options: # We don't need a full matrix
@@ -85,8 +83,6 @@ jobs:
8583
steps:
8684
- name: Checkout
8785
uses: actions/checkout@v4
88-
- name: Setup Environment
89-
run: brew install python@3.9
9086
- name: Checkout submodules & Bootstrap
9187
uses: ./.github/actions/checkout-submodules-and-bootstrap
9288
with:
@@ -120,6 +116,7 @@ jobs:
120116
OTHER_CFLAGS='${inherited} -Werror -Wconversion' CHIP_IS_BLE=NO GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1 ${{ matrix.options.defines }}' \
121117
> >(tee /tmp/darwin/framework-tests/darwin-tests.log) 2> >(tee /tmp/darwin/framework-tests/darwin-tests-err.log >&2)
122118
- name: Collect crash logs
119+
if: failure() && !env.ACT
123120
run: |
124121
mkdir -p /tmp/darwin/framework-tests
125122
find ~/Library/Developer/Xcode/DerivedData /Library/Logs/DiagnosticReports -name '*.ips' -print0 | xargs -0 -J % cp % /tmp/darwin/framework-tests
@@ -134,24 +131,15 @@ jobs:
134131
tv-casting-bridge:
135132
name: Build TV Casting Bridge example
136133
if: github.actor != 'restyled-io[bot]'
137-
runs-on: macos-latest
134+
needs: [ framework ] # serialize to avoid running to many parallel macos runners
135+
runs-on: macos-13
138136
steps:
139137
- name: Checkout
140138
uses: actions/checkout@v4
141-
- name: Setup Environment
142-
run: brew install python@3.9
143139
- name: Checkout submodules & Bootstrap
144140
uses: ./.github/actions/checkout-submodules-and-bootstrap
145141
with:
146142
platform: darwin
147143
- name: Build
148144
working-directory: examples/tv-casting-app/darwin/MatterTvCastingBridge
149145
run: xcodebuild -target "MatterTvCastingBridge" -sdk iphoneos
150-
151-
darwin:
152-
name: Build Darwin # Matches the previous monolithic build that's marked "required" for PRs
153-
needs: [ framework, tests ]
154-
runs-on: macos-latest
155-
steps:
156-
- name: Done
157-
run: 'true' # nothing to do

‎scripts/dump_diskspace_info.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ echo
3636
echo "Storage Space Used By Key Directories:"
3737
for directory in "${listOfDirectories[@]}"; do
3838
if [ -d "$directory" ]; then
39-
du -d1 -h "$directory" | sort -h
39+
du -d1 -h "$directory" | grep '^[[:space:]]*[0-9.]*[MG][[:space:]]' | sort -rh
4040
echo
4141
fi
4242
done

0 commit comments

Comments
 (0)