Merge branch 'master' into feature/da_verifier_revocation_check_2

Author: vijs

.flake8

@@ -369,6 +369,7 @@ DelayedActionTime
 delayedApplyActionTimeSec
 delayedQueryActionTimeSec
 delayQuery
+deliverables
 demangle
 deployable
 depottools
@@ -585,6 +586,8 @@ GenericWiFiConfigurationManagerImpl
 GetDeviceId
 GetDeviceInfo
 GetDns
+getter
+getters
 GetInDevelopmentTests
 GetIP
 getManualTests
@@ -643,6 +646,8 @@ HomePods
 hostapd
 hostname
 href
+HSM
+hsm
 HTTPS
 HW
 hwadr
@@ -668,6 +673,7 @@ ifdef
 ifdefs
 IGMP
 ihex
+Illuminance
 IlluminanceMeasurement
 IM
 imager
@@ -724,6 +730,7 @@ JLink
 JLinkExe
 JLinkRTTClient
 JN
+jni
 jpg
 jre
 js
@@ -904,6 +911,7 @@ Multicast
 multilib
 Multiprotocol
 multithreaded
+mutex
 mutexes
 mv
 MX
@@ -933,6 +941,7 @@ nfds
 NitricOxideConcentrationMeasurement
 NitrogenDioxideConcentrationMeasurement
 nl
+nltest
 NLUnitTest
 NLUnitTests
 nmcli
@@ -964,6 +973,7 @@ objcopy
 OccupancySensing
 OctetString
 OECORE
+OID
 ol
 Onboarding
 onboardingcodes
@@ -985,6 +995,7 @@ openweave
 OperationalCredentials
 operationalDataset
 opkg
+OPTIGA
 optionMask
 optionOverride
 optionsMask
@@ -1428,6 +1439,7 @@ transitionTime
 TransportMgrBase
 TriggerEffect
 TRNG
+trustm
 TrustedRootCertificates
 tsan
 TSG
@@ -1493,6 +1505,7 @@ utils
 UUID
 ux
 validator
+valgrind
 vcom
 VCP
 Vectorcall

.github/.wordlist.txt

@@ -1,9 +1,9 @@
 name: Bootstrap cache
-description: Bootstrap cache
+description: Bootstrap cache (deprecated)
 runs:
   using: "composite"
   steps:
-    - uses: Wandalen/wretry.action@v1.3.0
+    - uses: Wandalen/wretry.action@v1.4.10
       name: Bootstrap cache
       continue-on-error: true
       with:

.github/actions/bootstrap-cache/action.yaml

@@ -5,9 +5,73 @@ inputs:
     description: "Platform name"
     required: false
     default: none
+  bootstrap-log-name:
+    description: "Bootstrap log name"
+    required: false
+    default: bootstrap-logs-${{ github.job }}
+outputs:
+  cache-hit:
+    description: "Bootstrap environment was restored from cache"
+    value: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit }} # dynamic action returns all outputs in `outputs`
+
 runs:
   using: "composite"
   steps:
-    - name: Bootstrap
+    - name: Determine bootstrap cache configuration
+      id: prepare
+      shell: bash
+      run: |
+        # Determine bootstrap cache configuration
+        # In addition to the various setup files, the work directory matters as well,
+        # because the bootstrapped Pigweed environment contains absolute paths.
+        echo "Calculating bootstrap cache key for '$PWD'"
+        FILES_HASH="${{ hashFiles('scripts/setup/*', 'third_party/pigweed/**') }}"
+        FINAL_HASH="$(echo "$PWD:$FILES_HASH" | shasum -a 256 | cut -d' ' -f1)"
+        echo key="${RUNNER_OS}-${RUNNER_ARCH}-${{ inputs.platform }}-${FINAL_HASH}" | tee -a "$GITHUB_OUTPUT"
+
+        # Split caches across backends
+        case "$RUNNER_OS" in
+          macOS) echo backend=actions;;
+          *)     echo backend=buildjet;;
+        esac | tee -a "$GITHUB_OUTPUT"
+
+    - name: Bootstrap from cache
+      id: restore
+      uses: ./.github/actions/dynamic
+      continue-on-error: true
+      with:
+        action: ${{ steps.prepare.outputs.backend }}/cache/restore@v4
+        with: |
+          key: ${{ steps.prepare.outputs.key }}
+          path: |
+            .environment
+            build_overrides/pigweed_environment.gni
+
+    - name: Run bootstrap
+      if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+      env:
+        PW_NO_CIPD_CACHE_DIR: 1
+        PW_ENVSETUP_NO_BANNER: 1
       shell: bash
-      run: bash scripts/bootstrap.sh -p all,${{ inputs.platform }}
+      run: source scripts/bootstrap.sh -p all,${{ inputs.platform }}
+
+    - name: Save bootstrap cache
+      uses: ./.github/actions/dynamic
+      if: ${{ fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+      continue-on-error: true
+      with:
+        action: ${{ steps.prepare.outputs.backend }}/cache/save@v4
+        with: |
+          key: ${{ steps.prepare.outputs.key }}
+          path: |
+            .environment
+            build_overrides/pigweed_environment.gni
+
+    - name: Upload bootstrap logs
+      uses: actions/upload-artifact@v4
+      if: ${{ always() && !env.ACT && fromJSON(steps.restore.outputs.outputs).cache-hit != 'true' }}
+      with:
+        name: ${{ inputs.bootstrap-log-name }}
+        path: |
+            .environment/gn_out/.ninja_log
+            .environment/pigweed-venv/*.log

.github/actions/bootstrap/action.yaml

@@ -11,7 +11,7 @@ inputs:
   bootstrap-log-name:
     description: "Bootstrap log name"
     required: false
-    default: bootstrap-logs
+    default: bootstrap-logs-${{ github.job }}
 runs:
   using: "composite"
   steps:
@@ -26,18 +26,18 @@ runs:
       with:
         platform: ${{ inputs.platform }}
         extra-parameters: ${{ inputs.extra-submodule-parameters }}
-    - name: Bootstrap Cache
-      uses: ./.github/actions/bootstrap-cache
     - name: Bootstrap
       uses: ./.github/actions/bootstrap
-      env:
-        PW_NO_CIPD_CACHE_DIR: Y
       with:
         platform: ${{ inputs.platform }}
+        bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
     - name: Dump disk info after checkout submodule & Bootstrap
       shell: bash
       run: scripts/dump_diskspace_info.sh
-    - name: Upload Bootstrap Logs
-      uses: ./.github/actions/upload-bootstrap-logs
-      with:
-        bootstrap-log-name: ${{ inputs.bootstrap-log-name }}
+    - name: Work around TSAN ASLR issues
+      if: runner.os == 'Linux' && !env.ACT
+      shell: bash
+      run: |
+        # See https://stackoverflow.com/a/77856955/2365113
+        if [[ "$UID" == 0 ]]; then function sudo() { "$@"; }; fi
+        sudo sysctl vm.mmap_rnd_bits=28

.github/actions/checkout-submodules-and-bootstrap/action.yaml

@@ -11,7 +11,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: Wandalen/wretry.action@v1.3.0
+    - uses: Wandalen/wretry.action@v1.4.10
       name: Checkout submodules
       with:
         command: scripts/checkout_submodules.py --allow-changing-global-git-config --shallow --platform ${{ inputs.platform }} ${{ inputs.extra-parameters }}

.github/actions/checkout-submodules/action.yaml

@@ -0,0 +1,38 @@
+name: Dynamic
+description: Dynamically resolves another GitHub action
+inputs:
+  action:
+    description: Action reference
+    required: true
+  with:
+    description: Action inputs as multi-line YAML string
+    required: false
+    default: ""
+outputs:
+  outputs:
+    description: JSON object of outputs from the action
+    value: ${{ steps.run.outputs.outputs }}
+runs:
+  using: composite
+  steps:
+    - name: Instantiate
+      shell: bash
+      run: |
+        # Dynamically invoke ${{ inputs.action }}
+        with='${{ inputs.with }}'
+        [[ -z "$with" ]] || with="$(echo '    with:'; sed -e 's/^/      /' <<<"$with")"
+        mkdir -p ./.tmp/dynamic-action-instance
+        cat <<END >./.tmp/dynamic-action-instance/action.yaml
+        runs:
+          using: composite
+          steps:
+          - id: run
+            uses: ${{ inputs.action }}
+        $with
+        outputs:
+          outputs:
+            value: $(echo '$'){{ toJSON(steps.run.outputs) }}
+        END
+    - name: Run
+      id: run
+      uses: ./.tmp/dynamic-action-instance

.github/actions/dynamic/action.yaml

@@ -4,47 +4,45 @@ runs:
   using: "composite"
   steps:
     - name: Free up disk space on the github runner
-      if: ${{ !env.ACT }}
+      if: runner.os == 'Linux' && !env.ACT
       shell: bash
       run: |
         # maximize-runner-disk
-        if [[ "$RUNNER_OS" == Linux ]]; then
-          # Directories to prune to free up space. Candidates:
-          #   1.6G  /usr/share/dotnet
-          #   1.1G  /usr/local/lib/android/sdk/platforms
-          #   1000M /usr/local/lib/android/sdk/build-tools
-          #   8.9G  /usr/local/lib/android/sdk
-          # This list can be amended later to change the trade-off between the amount of
-          # disk space freed up, and how long it takes to do so (deleting many files is slow).
-          prune=(/usr/share/dotnet /usr/local/lib/android/sdk/platforms /usr/local/lib/android/sdk/build-tools)
+        # Directories to prune to free up space. Candidates:
+        #   1.6G  /usr/share/dotnet
+        #   1.1G  /usr/local/lib/android/sdk/platforms
+        #   1000M /usr/local/lib/android/sdk/build-tools
+        #   8.9G  /usr/local/lib/android/sdk
+        # This list can be amended later to change the trade-off between the amount of
+        # disk space freed up, and how long it takes to do so (deleting many files is slow).
+        prune=(/usr/share/dotnet /usr/local/lib/android/sdk/platforms /usr/local/lib/android/sdk/build-tools)
 
-          if [[ "$UID" -eq 0 && -d /__w ]]; then
-            root=/runner-root-volume
-            if [[ ! -d "$root" ]]; then
-              echo "Unable to maximize disk space, job is running inside a container and $root is not mounted"
-              exit 0
-            fi
-            function sudo() { "$@"; } # we're already root (and sudo is probably unavailable)
-          elif [[ "$UID" -ne 0 && "$RUNNER_ENVIRONMENT" == github-hosted ]]; then
-            root=
-          else
-            echo "Unable to maximize disk space, unknown runner environment"
+        if [[ "$UID" -eq 0 && -d /__w ]]; then
+          root=/runner-root-volume
+          if [[ ! -d "$root" ]]; then
+            echo "Unable to maximize disk space, job is running inside a container and $root is not mounted"
             exit 0
           fi
-
-          echo "Freeing up runner disk space on ${root:-/}"
-          function avail() { df -k --output=avail "${root:-/}" | grep '^[0-9]*$'; }
-          function now() { date '+%s'; }
-          before="$(avail)" start="$(now)"
-          for dir in "${prune[@]}"; do
-            if [[ -d "${root}${dir}" ]]; then
-              echo "- $dir"
-              # du -sh -- "${root}${dir}"
-              sudo rm -rf -- "${root}${dir}"
-            else
-              echo "- $dir (not found)"
-            fi
-          done
-          after="$(avail)" end="$(now)"
-          echo "Done, freed up $(( (after - before) / 1024 ))M of disk space in $(( end - start )) seconds."
+          function sudo() { "$@"; } # we're already root (and sudo is probably unavailable)
+        elif [[ "$UID" -ne 0 && "$RUNNER_ENVIRONMENT" == github-hosted ]]; then
+          root=
+        else
+          echo "Unable to maximize disk space, unknown runner environment"
+          exit 0
         fi
+
+        echo "Freeing up runner disk space on ${root:-/}"
+        function avail() { df -k --output=avail "${root:-/}" | grep '^[0-9]*$'; }
+        function now() { date '+%s'; }
+        before="$(avail)" start="$(now)"
+        for dir in "${prune[@]}"; do
+          if [[ -d "${root}${dir}" ]]; then
+            echo "- $dir"
+            # du -sh -- "${root}${dir}"
+            sudo rm -rf -- "${root}${dir}"
+          else
+            echo "- $dir (not found)"
+          fi
+        done
+        after="$(avail)" end="$(now)"
+        echo "Done, freed up $(( (after - before) / 1024 ))M of disk space in $(( end - start )) seconds."

.github/actions/maximize-runner-disk/action.yaml

@@ -27,7 +27,7 @@ runs:
       with:
         sarif_file: "sarif-results/${{ inputs.language }}.sarif"
     - name: Upload loc as a Build Artifact
-      uses: actions/upload-artifact@v2.2.0
+      uses: actions/upload-artifact@v4
       with:
         name: sarif-results
         path: sarif-results

.github/actions/perform-codeql-analysis/action.yaml

@@ -9,7 +9,7 @@ runs:
   using: "composite"
   steps:
     - name: Uploading Size Reports
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: ${{ !env.ACT }}
       with:
         name: Size,${{ inputs.platform-name }}-Examples,${{ env.GH_EVENT_PR }},${{ env.GH_EVENT_HASH }},${{ env.GH_EVENT_PARENT }},${{ github.event_name }}