Merge branch 'master' into Set-3-20-24

Author: raul-marquez-csa

.github/actions/bootstrap-cache/action.yaml

@@ -1,5 +1,5 @@
 name: Bootstrap cache
-description: Bootstrap cache
+description: Bootstrap cache (deprecated)
 runs:
   using: "composite"
   steps:

.github/actions/bootstrap/action.yaml

@@ -5,9 +5,73 @@ inputs:
     description: "Platform name"
     required: false
     default: none
+  bootstrap-log-name:
+    description: "Bootstrap log name"
+    required: false
+    default: bootstrap-logs-${{ github.job }}
+outputs:
+  cache-hit:
+    description: "Bootstrap environment was restored from cache"
+    value: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit }} # dynamic action returns all outputs in `outputs`
+
 runs:
   using: "composite"
   steps:
-    - name: Bootstrap
+    - name: Determine bootstrap cache configuration
+      id: prepare
+      shell: bash
+      run: |
+        # Determine bootstrap cache configuration
+        # In addition to the various setup files, the work directory matters as well,
+        # because the bootstrapped Pigweed environment contains absolute paths.
+        echo "Calculating bootstrap cache key for '$PWD'"
+        FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
+        FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
+        echo key="${RUNNER_OS}-${RUNNER_ARCH}-${{ inputs.platform }}-${FINAL_HASH}" | tee -a "$GITHUB_OUTPUT"
+
+        # Split caches across backends
+        case "$RUNNER_OS" in
+          macOS) echo backend=actions;;
+          *)     echo backend=buildjet;;
+        esac | tee -a "$GITHUB_OUTPUT"
+
+    - name: Bootstrap from cache
+      id: restore
+      uses: ./.github/actions/dynamic
+      continue-on-error: true
+      with:
+        action: ${{ steps.prepare.outputs.backend }}/cache/restore@v4
+        with: |
+          key: ${{ steps.prepare.outputs.key }}
+          path: |
+            .environment
+            build_overrides/pigweed_environment.gni
+
+    - name: Run bootstrap
+      if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+      env:
+        PW_NO_CIPD_CACHE_DIR: 1
+        PW_ENVSETUP_NO_BANNER: 1
       shell: bash
-      run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }}
+      run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
+
+    - name: Save bootstrap cache
+      uses: ./.github/actions/dynamic
+      if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+      continue-on-error: true
+      with:
+        action: ${{ steps.prepare.outputs.backend }}/cache/save@v4
+        with: |
+          key: ${{ steps.prepare.outputs.key }}
+          path: |
+            .environment
+            build_overrides/pigweed_environment.gni
+
+    - name: Upload bootstrap logs
+      uses: actions/upload-artifact@v4
+      if: ${{ always() && !env.ACT && fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+      with:
+        name: ${{ inputs.bootstrap-log-name }}
+        path: |
+            .environment/gn_out/.ninja_log
+            .environment/pigweed-venv/*.log

.github/actions/checkout-submodules-and-bootstrap/action.yaml

@@ -26,21 +26,14 @@ runs:
       with:
         platform: ${{ inputs.platform }}
         extra-parameters: ${{ inputs.extra-submodule-parameters }}
-    - name: Bootstrap Cache
-      uses: ./.github/actions/bootstrap-cache
     - name: Bootstrap
       uses: ./.github/actions/bootstrap
-      env:
-        PW_NO_CIPD_CACHE_DIR: Y
       with:
         platform: ${{ inputs.platform }}
+        bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
     - name: Dump disk info after checkout submodule & Bootstrap
       shell: bash
       run: scripts/dump_diskspace_info.sh
-    - name: Upload Bootstrap Logs
-      uses: ./.github/actions/upload-bootstrap-logs
-      with:
-        bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
     - name: Work around TSAN ASLR issues
       if: runner.os == 'Linux' && !env.ACT
       shell: bash

.github/actions/dynamic/action.yaml

@@ -0,0 +1,38 @@
+name: Dynamic
+description: Dynamically resolves another GitHub action
+inputs:
+  action:
+    description: Action reference
+    required: true
+  with:
+    description: Action inputs as multi-line YAML string
+    required: false
+    default: ""
+outputs:
+  outputs:
+    description: JSON object of outputs from the action
+    value: ${{ steps.run.outputs.outputs }}
+runs:
+  using: composite
+  steps:
+    - name: Instantiate
+      shell: bash
+      run: |
+        # Dynamically invoke ${{ inputs.action }}
+        with='${{ inputs.with }}'
+        [[ -z "$with" ]] || with="$(echo '    with:'; sed -e 's/^/      /' <<<"$with")"
+        mkdir -p ./.tmp/dynamic-action-instance
+        cat <<END >./.tmp/dynamic-action-instance/action.yaml
+        runs:
+          using: composite
+          steps:
+          - id: run
+            uses: ${{ inputs.action }}
+        $with
+        outputs:
+          outputs:
+            value: $(echo '$'){{ toJSON(steps.run.outputs) }}
+        END
+    - name: Run
+      id: run
+      uses: ./.tmp/dynamic-action-instance

.github/actions/upload-bootstrap-logs/action.yaml

@@ -57,6 +57,7 @@ jobs:
               with:
                 platform: linux
 
+            # TODO: Is what's being cached here actually compatible with a regular bootstrap?
             - name: Bootstrap Cache
               uses: ./.github/actions/bootstrap-cache
             - name: Bootstrap Cirque

.github/workflows/cirque.yaml

@@ -49,7 +49,7 @@ jobs:
             LSAN_OPTIONS: detect_leaks=1 malloc_context_size=40 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt
 
         if: github.actor != 'restyled-io[bot]'
-        runs-on: macos-latest
+        runs-on: macos-13
 
         steps:
             - name: Checkout

.github/workflows/darwin-tests.yaml

@@ -31,34 +31,34 @@ jobs:
     framework:
         name: Build framework
         if: github.actor != 'restyled-io[bot]'
-        runs-on: macos-latest
+        runs-on: macos-13
         strategy:
             matrix:
                 options: # We don't need a full matrix
                     - flavor: macos-release-availability
-                      arguments: -sdk macosx -configuration Release   OTHER_CFLAGS='${inherited} -Werror -Wconversion -Wno-unguarded-availability-new'
+                      arguments: -sdk macosx -configuration Release   WARNING_CFLAGS='${inherited} -Werror -Wconversion -Wno-unguarded-availability-new'
                     - flavor: ios-release
-                      arguments: -sdk iphoneos -configuration Release OTHER_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
+                      arguments: -sdk iphoneos -configuration Release WARNING_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
                     - flavor: ios-debug
-                      arguments: -sdk iphoneos -configuration Debug   OTHER_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
+                      arguments: -sdk iphoneos -configuration Debug   WARNING_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
                     - flavor: tvos-debug
-                      arguments: -sdk appletvos -configuration Debug  OTHER_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
+                      arguments: -sdk appletvos -configuration Debug  WARNING_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
                     - flavor: watchos-debug
-                      arguments: -sdk watchos -configuration Debug    OTHER_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
+                      arguments: -sdk watchos -configuration Debug    WARNING_CFLAGS='${inherited} -Werror -Wconversion' GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1'
         steps:
             - name: Checkout
               uses: actions/checkout@v4
-            - name: Setup Environment
-              run: brew install python@3.9
             - name: Checkout submodules & Bootstrap
               uses: ./.github/actions/checkout-submodules-and-bootstrap
               with:
                 platform: darwin
                 bootstrap-log-name: bootstrap-logs-framework-${{ matrix.options.flavor }}
             - name: Block zap-cli from being used
+              env:
+                PW_ENVSETUP_NO_BANNER: 1
               run: |
                 # Framework builds are NOT expected to require zap-cli
-                scripts/run_in_build_env.sh 'D=$(dirname $(which zap-cli)) && mv $D/zap-cli $D/zap-cli.moved'
+                scripts/run_in_build_env.sh 'rm -- "$(which zap-cli)"'
                 # run_in_build_env.sh is used to ensure PATH is set to something that would otherwise find zap-cli
                 scripts/run_in_build_env.sh '(zap-cli --version && exit 1) || exit 0'
             - name: Build
@@ -68,7 +68,8 @@ jobs:
     tests:
         name: Run framework tests
         if: github.actor != 'restyled-io[bot]'
-        runs-on: macos-latest
+        needs: [ framework ] # serialize to avoid running to many parallel macos runners
+        runs-on: macos-13
         strategy:
             matrix:
                 options: # We don't need a full matrix
@@ -82,8 +83,6 @@ jobs:
         steps:
             - name: Checkout
               uses: actions/checkout@v4
-            - name: Setup Environment
-              run: brew install python@3.9
             - name: Checkout submodules & Bootstrap
               uses: ./.github/actions/checkout-submodules-and-bootstrap
               with:
@@ -117,6 +116,7 @@ jobs:
                     OTHER_CFLAGS='${inherited} -Werror -Wconversion' CHIP_IS_BLE=NO GCC_PREPROCESSOR_DEFINITIONS='${inherited} MTR_NO_AVAILABILITY=1 ${{ matrix.options.defines }}' \
                     > >(tee /tmp/darwin/framework-tests/darwin-tests.log) 2> >(tee /tmp/darwin/framework-tests/darwin-tests-err.log >&2)
             - name: Collect crash logs
+              if: failure() && !env.ACT
               run: |
                   mkdir -p /tmp/darwin/framework-tests
                   find ~/Library/Developer/Xcode/DerivedData /Library/Logs/DiagnosticReports -name '*.ips' -print0 | xargs -0 -J % cp % /tmp/darwin/framework-tests
@@ -131,24 +131,15 @@ jobs:
     tv-casting-bridge:
         name: Build TV Casting Bridge example
         if: github.actor != 'restyled-io[bot]'
-        runs-on: macos-latest
+        needs: [ framework ] # serialize to avoid running to many parallel macos runners
+        runs-on: macos-13
         steps:
             - name: Checkout
               uses: actions/checkout@v4
-            - name: Setup Environment
-              run: brew install python@3.9
             - name: Checkout submodules & Bootstrap
               uses: ./.github/actions/checkout-submodules-and-bootstrap
               with:
                 platform: darwin
             - name: Build
               working-directory: examples/tv-casting-app/darwin/MatterTvCastingBridge
               run: xcodebuild -target "MatterTvCastingBridge" -sdk iphoneos
-
-    darwin:
-        name: Build Darwin # Matches the previous monolithic build that's marked "required" for PRs
-        needs: [ framework, tests ]
-        runs-on: macos-latest
-        steps:
-            - name: Done
-              run: 'true' # nothing to do

.github/workflows/darwin.yaml

@@ -59,15 +59,14 @@ jobs:
         timeout-minutes: 30
         run: |
           scripts/examples/gn_silabs_example.sh examples/lighting-app/silabs ./out/light-app BRD4187C --slc_generate --docker
-          scripts/examples/gn_silabs_example.sh examples/lighting-app/silabs ./out/light-app BRD4164A --slc_generate --docker
           rm -rf ./out/
       - name: Build some BRD4187C variants (1)
         run: |
           ./scripts/run_in_build_env.sh \
              "./scripts/build/build_examples.py \
                 --enable-flashbundle \
-                --target efr32-brd4187c-thermostat-openthread_mtd \
-                --target efr32-brd4187c-switch-shell-use_ot_coap_lib \
+                --target efr32-brd4187c-thermostat-openthread-mtd \
+                --target efr32-brd4187c-switch-shell-use-ot-coap-lib \
                 --target efr32-brd4187c-unit-test \
                 build \
                 --copy-artifacts-to out/artifacts \
@@ -79,9 +78,9 @@ jobs:
           ./scripts/run_in_build_env.sh \
              "./scripts/build/build_examples.py \
                 --enable-flashbundle \
-                --target efr32-brd4187c-light-use_ot_lib \
+                --target efr32-brd4187c-light-use-ot-lib \
                 --target efr32-brd4187c-pump \
-                --target efr32-brd4187c-lock-shell-enable_heap_monitoring \
+                --target efr32-brd4187c-lock-shell-heap-monitoring \
                 build \
                 --copy-artifacts-to out/artifacts \
              "
@@ -92,7 +91,7 @@ jobs:
           ./scripts/run_in_build_env.sh \
              "./scripts/build/build_examples.py \
                 --enable-flashbundle \
-                --target efr32-brd4187c-window-covering-additional_data_advertising \
+                --target efr32-brd4187c-window-covering-additional-data-advertising \
                 --target efr32-brd4187c-light-rpc \
                 build \
                 --copy-artifacts-to out/artifacts \
@@ -105,33 +104,39 @@ jobs:
              /tmp/bloat_reports/
           .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py  \
              efr32 BRD4187C window-app \
-             out/efr32-brd4187c-window-covering-additional_data_advertising/matter-silabs-window-example.out \
+             out/efr32-brd4187c-window-covering-additional-data-advertising/matter-silabs-window-example.out \
              /tmp/bloat_reports/
       - name: Clean out build output
         run: rm -rf ./out
-      - name: Build BRD4338A variants
+      - name: Build BRD4338A WiFi Soc variants
         run: |
           ./scripts/run_in_build_env.sh \
              "./scripts/build/build_examples.py \
                 --enable-flashbundle \
-                --target efr32-brd4338a-light-wifi-917_soc-skip_rps_generation \
+                --target efr32-brd4338a-light-skip-rps-generation \
+                --target efr32-brd4338a-lock-skip-rps-generation \
                 build \
                 --copy-artifacts-to out/artifacts \
              "
-      - name: Clean out build output
-        run: rm -rf ./out
-      - name: Build example EFR32+WF200 WiFi Lock app for BRD4161A
+      - name: Prepare bloat report for brd4338a lock app
         run: |
-          scripts/examples/gn_silabs_example.sh examples/lock-app/silabs out/lock_app_wifi_wf200 BRD4161A is_debug=false chip_logging=false --wifi wf200 --docker
-          .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py efr32 BRD4161A+wf200 lock-app \
-            out/lock_app_wifi_wf200/BRD4161A/matter-silabs-lock-example.out /tmp/bloat_reports/
+          .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \
+            efr32 BRD4338a lock-app \
+            out/efr32-brd4338a-lock-skip-rps-generation/matter-silabs-lock-example.out \
+            /tmp/bloat_reports/
       - name: Clean out build output
         run: rm -rf ./out
-      - name: Build example EFR32+RS9116 WiFi Lighting app for BRD4161A
+      - name: Build EFR32 with WiFi NCP
         run: |
-          scripts/examples/gn_silabs_example.sh examples/lighting-app/silabs out/lighting_app_wifi_rs9116 BRD4161A --wifi rs9116 --docker
-          .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py efr32 BRD4161A+rs9116 lighting-app \
-            out/lighting_app_wifi_rs9116/BRD4161A/matter-silabs-lighting-example.out /tmp/bloat_reports/
+          ./scripts/run_in_build_env.sh \
+            "./scripts/build/build_examples.py \
+                --enable-flashbundle \
+                --target efr32-brd4187c-lock-wifi-siwx917 \
+                --target efr32-brd4187c-light-wifi-rs9116 \
+                --target efr32-brd4187c-lock-wifi-wf200 \
+                build \
+                --copy-artifacts-to out/artifacts \
+            "
       - name: Clean out build output
         run: rm -rf ./out
       - name: Uploading Size Reports