tests.yaml

# Copyright (c) 2020-2021 Project CHIP Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Tests

on:
    push:
        branches:
            - master
            - 'v*-branch'
    pull_request:
    merge_group:
    workflow_dispatch:

concurrency:
    group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name ==
        'pull_request' && github.event.number) || (github.event_name ==
        'workflow_dispatch' && github.run_number) || github.sha }}
    cancel-in-progress: true

env:
    CHIP_NO_LOG_TIMESTAMPS: true

jobs:
    test_suites_linux:
        name: Test Suites - Linux

        strategy:
            matrix:
                build_variant: [no-ble-no-shell-tsan-clang]
                chip_tool: [""]
        env:
            BUILD_VARIANT: ${{matrix.build_variant}}
            CHIP_TOOL_VARIANT: ${{matrix.chip_tool}}
            TSAN_OPTIONS: "halt_on_error=1 suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt"
            LSAN_OPTIONS: detect_leaks=1

        if: github.actor != 'restyled-io[bot]'
        runs-on: ubuntu-latest

        container:
            image: ghcr.io/project-chip/chip-build:119
            options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
                net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1"

        steps:
            - name: Checkout
              uses: actions/checkout@v4
            # To use act like:
            #   act -j test_suites_linux
            #
            # Note you likely still need to have non submodules setup for the
            # local machine, like:
            #   git submodule deinit --all
            - uses: actions/checkout@v4
              if: ${{ env.ACT }}
              name: Checkout (ACT for local build)
            - name: Checkout submodules & Bootstrap
              uses: ./.github/actions/checkout-submodules-and-bootstrap
              with:
                platform: linux
                bootstrap-log-name: bootstrap-logs-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }}
            - name: Try to ensure the directories for core dumping exist and we
                  can write them.
              run: |
                  mkdir /tmp/cores || true
                  sysctl -w kernel.core_pattern=/tmp/cores/core.%u.%p.%t || true
                  mkdir objdir-clone || true

            - name: Validate that xml are parsable
              # The sub-items being run here are the same as the input XMLs listed
              # at src/app/zap-templates/zcl/zcl.json
              #
              # This ensures that the syntax of the XML can always be parsed/validated, however it
              # does not enforce that the content is understood (that part is covered by parser
              # unit tests)
              #
              run: |
                  ./scripts/run_in_build_env.sh \
                    "./scripts/py_matter_idl/matter_idl/zapxml_parser.py \
                      --no-print \
                      --log-level info \
                      src/app/zap-templates/zcl/data-model/chip/global-attributes.xml \
                      src/app/zap-templates/zcl/data-model/chip/chime-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/closure-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/global-bitmaps.xml \
                      src/app/zap-templates/zcl/data-model/chip/global-enums.xml \
                      src/app/zap-templates/zcl/data-model/chip/global-structs.xml \
                      src/app/zap-templates/zcl/data-model/chip/meter-identification-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/push-av-stream-transport-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/semantic-tag-namespace-enums.xml \
                      src/app/zap-templates/zcl/data-model/chip/access-control-definitions.xml \
                      src/app/zap-templates/zcl/data-model/chip/access-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/account-login-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/administrator-commissioning-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/air-quality-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/application-basic-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/application-launcher-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/audio-output-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/ballast-configuration-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/basic-information-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/binding-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/boolean-state-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/actions-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/bridged-device-basic-information.xml \
                      src/app/zap-templates/zcl/data-model/chip/camera-av-settings-user-level-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/camera-av-stream-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/chip-ota.xml \
                      src/app/zap-templates/zcl/data-model/chip/chip-types.xml \
                      src/app/zap-templates/zcl/data-model/chip/channel-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/clusters-extensions.xml \
                      src/app/zap-templates/zcl/data-model/chip/color-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/commissioner-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/concentration-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/content-launch-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/content-app-observer-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/content-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/descriptor-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/device-energy-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/device-energy-management-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/diagnostic-logs-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/dishwasher-alarm-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/dishwasher-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/measurement-and-sensing.xml \
                      src/app/zap-templates/zcl/data-model/chip/microwave-oven-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/microwave-oven-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/door-lock-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/ecosystem-information-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/energy-evse-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/energy-evse-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/ethernet-network-diagnostics-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/fan-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/fault-injection-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/fixed-label-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/flow-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/general-commissioning-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/general-diagnostics-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/global-attributes.xml \
                      src/app/zap-templates/zcl/data-model/chip/groups-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/group-key-mgmt-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/identify-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/illuminance-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/keypad-input-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/laundry-washer-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/laundry-dryer-controls-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/level-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/localization-configuration-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/low-power-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/media-input-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/media-playback-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/messages-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/mode-base-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/mode-select-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/network-commissioning-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/occupancy-sensing-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/onoff-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/operational-credentials-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/operational-state-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/operational-state-oven-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/operational-state-rvc-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/oven-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/power-topology-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/pressure-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/power-source-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/power-source-configuration-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/proxy-configuration-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/proxy-discovery-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/proxy-valid-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/pump-configuration-and-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/pwm-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/refrigerator-and-temperature-controlled-cabinet-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/refrigerator-alarm.xml \
                      src/app/zap-templates/zcl/data-model/chip/relative-humidity-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/rvc-clean-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/rvc-run-mode-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/scene.xml \
                      src/app/zap-templates/zcl/data-model/chip/smoke-co-alarm-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/software-diagnostics-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/switch-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/target-navigator-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/temperature-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/temperature-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/test-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/thermostat-user-interface-configuration-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/thermostat-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/thread-border-router-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/thread-network-diagnostics-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/thread-network-directory-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/time-format-localization-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/time-synchronization-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/timer-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/tls-certificate-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/user-label-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/unit-localization-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/wake-on-lan-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/washer-controls-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/water-heater-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/webrtc-requestor-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/webrtc-provider-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/wifi-network-diagnostics-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/wifi-network-management-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/window-covering.xml \
                      src/app/zap-templates/zcl/data-model/chip/temperature-control-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/matter-devices.xml \
                      src/app/zap-templates/zcl/data-model/chip/resource-monitoring-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/sample-mei-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/electrical-energy-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/electrical-power-measurement-cluster.xml \
                      src/app/zap-templates/zcl/data-model/chip/zone-management-cluster.xml \
                    "
            - name: Build Apps
              run: |
                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv'
                  ./scripts/run_in_build_env.sh \
                     "./scripts/build/build_examples.py \
                        --target linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \
                        --target linux-x64-all-clusters-${BUILD_VARIANT} \
                        --target linux-x64-lock-${BUILD_VARIANT} \
                        --target linux-x64-ota-provider-${BUILD_VARIANT} \
                        --target linux-x64-ota-requestor-${BUILD_VARIANT} \
                        --target linux-x64-tv-app-${BUILD_VARIANT} \
                        --target linux-x64-bridge-${BUILD_VARIANT} \
                        --target linux-x64-lit-icd-${BUILD_VARIANT} \
                        --target linux-x64-microwave-oven-${BUILD_VARIANT} \
                        --target linux-x64-rvc-${BUILD_VARIANT} \
                        --target linux-x64-network-manager-${BUILD_VARIANT} \
                        build \
                        --copy-artifacts-to objdir-clone \
                     "

            - name: Run Tests using the python parser sending commands to chip-tool
              run: |
                  ./scripts/run_in_build_env.sh \
                  "./scripts/tests/run_test_suite.py \
                     --runner chip_tool_python \
                     --chip-tool ./out/linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \
                     run \
                     --iterations 1 \
                     --test-timeout-seconds 120 \
                     --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \
                     --lock-app ./out/linux-x64-lock-${BUILD_VARIANT}/chip-lock-app \
                     --ota-provider-app ./out/linux-x64-ota-provider-${BUILD_VARIANT}/chip-ota-provider-app \
                     --ota-requestor-app ./out/linux-x64-ota-requestor-${BUILD_VARIANT}/chip-ota-requestor-app \
                     --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \
                     --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \
                     --lit-icd-app ./out/linux-x64-lit-icd-${BUILD_VARIANT}/lit-icd-app \
                     --microwave-oven-app ./out/linux-x64-microwave-oven-${BUILD_VARIANT}/chip-microwave-oven-app \
                     --rvc-app ./out/linux-x64-rvc-${BUILD_VARIANT}/chip-rvc-app \
                     --network-manager-app ./out/linux-x64-network-manager-${BUILD_VARIANT}/matter-network-manager-app \
                  "

            - name: Run purposeful failure tests using the python parser sending commands to chip-tool
              run: |
                  ./scripts/run_in_build_env.sh \
                  "./scripts/tests/run_test_suite.py \
                     --runner chip_tool_python \
                     --include-tags PURPOSEFUL_FAILURE \
                     --chip-tool ./out/linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \
                     run \
                     --iterations 1 \
                     --expected-failures 3 \
                     --keep-going \
                     --test-timeout-seconds 120 \
                     --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \
                  "

            - name: Run Tests using chip-repl (skip slow)
              if: github.event_name == 'pull_request'
              run: |
                  ./scripts/run_in_python_env.sh out/venv \
                  "./scripts/tests/run_test_suite.py \
                     --runner chip_repl_python \
                     --exclude-tags MANUAL \
                     --exclude-tags FLAKY \
                     --exclude-tags IN_DEVELOPMENT \
                     --exclude-tags EXTRA_SLOW \
                     --exclude-tags SLOW \
                     --exclude-tags PURPOSEFUL_FAILURE \
                     run \
                     --iterations 1 \
                     --test-timeout-seconds 120 \
                     --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \
                     --lock-app ./out/linux-x64-lock-${BUILD_VARIANT}/chip-lock-app \
                     --ota-provider-app ./out/linux-x64-ota-provider-${BUILD_VARIANT}/chip-ota-provider-app \
                     --ota-requestor-app ./out/linux-x64-ota-requestor-${BUILD_VARIANT}/chip-ota-requestor-app \
                     --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \
                     --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \
                     --lit-icd-app ./out/linux-x64-lit-icd-${BUILD_VARIANT}/lit-icd-app \
                     --microwave-oven-app ./out/linux-x64-microwave-oven-${BUILD_VARIANT}/chip-microwave-oven-app \
                     --rvc-app ./out/linux-x64-rvc-${BUILD_VARIANT}/chip-rvc-app \
                     --network-manager-app ./out/linux-x64-network-manager-${BUILD_VARIANT}/matter-network-manager-app \
                  "
            - name: Run Tests using chip-repl (including slow)
              if: github.event_name == 'push'
              run: |
                  ./scripts/run_in_python_env.sh out/venv \
                  "./scripts/tests/run_test_suite.py \
                     --runner chip_repl_python \
                     run \
                     --iterations 1 \
                     --test-timeout-seconds 120 \
                     --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \
                     --lock-app ./out/linux-x64-lock-${BUILD_VARIANT}/chip-lock-app \
                     --ota-provider-app ./out/linux-x64-ota-provider-${BUILD_VARIANT}/chip-ota-provider-app \
                     --ota-requestor-app ./out/linux-x64-ota-requestor-${BUILD_VARIANT}/chip-ota-requestor-app \
                     --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \
                     --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \
                     --lit-icd-app ./out/linux-x64-lit-icd-${BUILD_VARIANT}/lit-icd-app \
                     --microwave-oven-app ./out/linux-x64-microwave-oven-${BUILD_VARIANT}/chip-microwave-oven-app \
                     --rvc-app ./out/linux-x64-rvc-${BUILD_VARIANT}/chip-rvc-app \
                     --network-manager-app ./out/linux-x64-network-manager-${BUILD_VARIANT}/matter-network-manager-app \
                  "
            - name: Uploading core files
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-core-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }}
                  path: /tmp/cores/
                  # Cores are big; don't hold on to them too long.
                  retention-days: 5
            - name: Uploading objdir for debugging
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-objdir-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }}
                  path: objdir-clone/
                  # objdirs are big; don't hold on to them too long.
                  retention-days: 5

    test_suites_darwin:
        name: Test Suites - Darwin

        strategy:
            matrix:
                build_variant: [no-ble-no-shell-asan-clang, no-ble-no-shell-tsan-clang]
                chip_tool: [""]
        env:
            BUILD_VARIANT: ${{matrix.build_variant}}
            CHIP_TOOL_VARIANT: ${{matrix.chip_tool}}
            TSAN_OPTIONS: "halt_on_error=1"
            LSAN_OPTIONS: detect_leaks=1 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt

        if: github.actor != 'restyled-io[bot]'
        runs-on: macos-13

        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Setup Environment
              # coreutils for stdbuf
              run: brew install coreutils
            - name:
                  Try to ensure the directories for core dumping and diagnostic
                  log collection exist and we can write them.
              run: |
                  sudo chown ${USER} /cores || true
                  mkdir -p ~/Library/Logs/DiagnosticReports || true
                  mkdir objdir-clone || true
            - name: Checkout submodules & Bootstrap
              uses: ./.github/actions/checkout-submodules-and-bootstrap
              with:
                platform: darwin
                bootstrap-log-name: bootstrap-logs-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }}


            - name: Build Apps
              run: |
                  ./scripts/run_in_build_env.sh \
                     "./scripts/build/build_examples.py \
                        --target darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \
                        --target darwin-x64-all-clusters-${BUILD_VARIANT} \
                        --target darwin-x64-lock-${BUILD_VARIANT} \
                        --target darwin-x64-ota-provider-${BUILD_VARIANT} \
                        --target darwin-x64-ota-requestor-${BUILD_VARIANT} \
                        --target darwin-x64-tv-app-${BUILD_VARIANT} \
                        --target darwin-x64-bridge-${BUILD_VARIANT} \
                        --target darwin-x64-lit-icd-${BUILD_VARIANT} \
                        --target darwin-x64-microwave-oven-${BUILD_VARIANT} \
                        --target darwin-x64-rvc-${BUILD_VARIANT} \
                        --target darwin-x64-network-manager-${BUILD_VARIANT} \
                        build \
                        --copy-artifacts-to objdir-clone \
                     "

            - name: Run Tests using the python parser sending commands to chip-tool
              run: |
                  ./scripts/run_in_build_env.sh \
                  "./scripts/tests/run_test_suite.py \
                     --runner chip_tool_python \
                     --chip-tool ./out/darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \
                     --target-skip-glob '{Test_TC_DGTHREAD_2_1,Test_TC_DGTHREAD_2_2,Test_TC_DGTHREAD_2_3,Test_TC_DGTHREAD_2_4}' \
                     run \
                     --iterations 1 \
                     --test-timeout-seconds 120 \
                     --all-clusters-app ./out/darwin-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \
                     --lock-app ./out/darwin-x64-lock-${BUILD_VARIANT}/chip-lock-app \
                     --ota-provider-app ./out/darwin-x64-ota-provider-${BUILD_VARIANT}/chip-ota-provider-app \
                     --ota-requestor-app ./out/darwin-x64-ota-requestor-${BUILD_VARIANT}/chip-ota-requestor-app \
                     --tv-app ./out/darwin-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \
                     --bridge-app ./out/darwin-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \
                     --lit-icd-app ./out/darwin-x64-lit-icd-${BUILD_VARIANT}/lit-icd-app \
                     --microwave-oven-app ./out/darwin-x64-microwave-oven-${BUILD_VARIANT}/chip-microwave-oven-app \
                     --rvc-app ./out/darwin-x64-rvc-${BUILD_VARIANT}/chip-rvc-app \
                     --network-manager-app ./out/darwin-x64-network-manager-${BUILD_VARIANT}/matter-network-manager-app \
                  "

            - name: Run purposeful failure tests using the python parser sending commands to chip-tool
              run: |
                  ./scripts/run_in_build_env.sh \
                  "./scripts/tests/run_test_suite.py \
                     --runner chip_tool_python \
                     --include-tags PURPOSEFUL_FAILURE \
                     --chip-tool ./out/darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \
                     run \
                     --iterations 1 \
                     --expected-failures 3 \
                     --keep-going \
                     --test-timeout-seconds 120 \
                     --all-clusters-app ./out/darwin-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \
                  "

            - name: Uploading core files
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-core-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }}
                  path: /cores/
                  # Cores are big; don't hold on to them too long.
                  retention-days: 5
            - name: Uploading diagnostic logs
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-log-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }}
                  path: ~/Library/Logs/DiagnosticReports/
            - name: Uploading objdir for debugging
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-objdir-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }}
                  path: objdir-clone/
                  # objdirs are big; don't hold on to them too long.
                  retention-days: 5

    repl_tests_linux:
        name: REPL Tests - Linux

        env:
            TSAN_OPTIONS: "halt_on_error=1 suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt"

        if: github.actor != 'restyled-io[bot]'
        runs-on: ubuntu-latest

        container:
            image: ghcr.io/project-chip/chip-build:119
            options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
                net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0"

        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Checkout submodules & Bootstrap
              uses: ./.github/actions/checkout-submodules-and-bootstrap
              with:
                platform: linux
                bootstrap-log-name: bootstrap-logs-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }}
            - name: Try to ensure the directories for core dumping exist and we
                  can write them.
              run: |
                  mkdir /tmp/cores || true
                  sysctl -w kernel.core_pattern=/tmp/cores/core.%u.%p.%t || true
                  mkdir objdir-clone || true

            - name: Build Python REPL and example apps
              # NOTE: the data-mode-check + check-failure-die is not 100% perfect as different
              #       encoding sizes for data that keeps changing may alter over time (e.g. anything relating to time
              #       or resources such as packet counts or other similar counters)
              #
              #       This may result in invalid errors, however for most purposes of our testing, we are unlikely to
              #       hit such cases so we remain very strict on testing here.
              run: |
                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv'
                  ./scripts/run_in_build_env.sh \
                   "./scripts/build/build_examples.py \
                      --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-lock-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-lit-icd-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-energy-management-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-microwave-oven-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-network-manager-ipv6only-no-ble-no-wifi-tsan-clang-test \
                      --target linux-x64-fabric-admin-rpc-ipv6only-no-ble-no-wifi-clang \
                      --target linux-x64-fabric-bridge-rpc-ipv6only-no-ble-no-wifi-clang \
                      --target linux-x64-fabric-sync-ipv6only-no-ble-no-wifi-clang \
                      --target linux-x64-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang \
                      --target linux-x64-terms-and-conditions \
                      --target linux-x64-python-bindings \
                      build \
                      --copy-artifacts-to objdir-clone \
                   "
            - name: Generate an argument environment file
              run: |
                  echo -n "" >/tmp/test_env.yaml
                  echo "ALL_CLUSTERS_APP: out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app" >> /tmp/test_env.yaml
                  echo "CHIP_LOCK_APP: out/linux-x64-lock-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-lock-app" >> /tmp/test_env.yaml
                  echo "ENERGY_MANAGEMENT_APP: out/linux-x64-energy-management-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-energy-management-app" >> /tmp/test_env.yaml
                  echo "LIT_ICD_APP: out/linux-x64-lit-icd-ipv6only-no-ble-no-wifi-tsan-clang-test/lit-icd-app" >> /tmp/test_env.yaml
                  echo "CHIP_MICROWAVE_OVEN_APP: out/linux-x64-microwave-oven-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-microwave-oven-app" >> /tmp/test_env.yaml
                  echo "CHIP_RVC_APP: out/linux-x64-rvc-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-rvc-app" >> /tmp/test_env.yaml
                  echo "NETWORK_MANAGEMENT_APP: out/linux-x64-network-manager-ipv6only-no-ble-no-wifi-tsan-clang-test/matter-network-manager-app" >> /tmp/test_env.yaml
                  echo "FABRIC_ADMIN_APP: out/linux-x64-fabric-admin-rpc-ipv6only-no-ble-no-wifi-clang/fabric-admin" >> /tmp/test_env.yaml
                  echo "FABRIC_BRIDGE_APP: out/linux-x64-fabric-bridge-rpc-ipv6only-no-ble-no-wifi-clang/fabric-bridge-app" >> /tmp/test_env.yaml
                  echo "FABRIC_SYNC_APP: out/linux-x64-fabric-sync-ipv6only-no-ble-no-wifi-clang/fabric-sync" >> /tmp/test_env.yaml
                  echo "LIGHTING_APP_NO_UNIQUE_ID: out/linux-x64-light-data-model-no-unique-id-ipv6only-no-ble-no-wifi-clang/chip-lighting-app" >> /tmp/test_env.yaml
                  echo "TERMS_AND_CONDITIONS_APP: out/linux-x64-terms-and-conditions/chip-terms-and-conditions-app" >> /tmp/test_env.yaml
                  echo "TRACE_APP: out/trace_data/app-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
                  echo "TRACE_TEST_JSON: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml
                  echo "TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml

            - name: Verify Testing Support
              run: |
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_IDM_10_4.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_ICDM_2_1_full_pics.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_ICDM_2_1_min_pics.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/test_TC_SC_7_1.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/test_testing/TestDecorators.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestChoiceConformanceSupport.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestConformanceSupport.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestConformanceTest.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestIdChecks.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestMatterTestingSupport.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingDeviceType.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingSelection.py'
                  scripts/run_in_python_env.sh out/venv 'python3 src/python_testing/TestSpecParsingSupport.py'

            - name: Run Tests
              run: |
                  mkdir -p out/trace_data
                  scripts/run_in_python_env.sh out/venv 'scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/controller/python/test/test_scripts/mobile-device-test.py'
                  scripts/run_in_python_env.sh out/venv 'src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
                  scripts/run_in_python_env.sh out/venv 'scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py --all-clusters out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app'

            - name: Execute Jupyter Notebooks
              run: |
                  scripts/run_in_build_env.sh './scripts/build_python.sh --jupyter-lab --install_virtual_env out/venv'
                  ./scripts/run_in_build_env.sh \
                   "./scripts/build/build_examples.py \
                      --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-clang-test \
                      build \
                      --copy-artifacts-to objdir-clone \
                   "
                  scripts/run_in_python_env.sh out/venv \
                  "jupyter execute docs/development_controllers/chip-repl/Matter_REPL_Intro.ipynb \
                  docs/development_controllers/chip-repl/Matter_Basic_Interactions.ipynb \
                  docs/development_controllers/chip-repl/Matter_Access_Control.ipynb \
                  docs/development_controllers/chip-repl/Matter_Multi_Fabric_Commissioning.ipynb \
                  "

            - name: Uploading core files
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-core-linux-python-repl
                  path: /tmp/cores/
                  # Cores are big; don't hold on to them too long.
                  retention-days: 5
            - name: Uploading objdir for debugging
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-objdir-linux-python-repl
                  path: objdir-clone/
                  # objdirs are big; don't hold on to them too long.
                  retention-days: 5

    repl_tests_darwin:
        name: REPL Tests - Darwin

        strategy:
            matrix:
                build_variant: [no-ble-no-wifi-tsan-clang]
        env:
            BUILD_VARIANT: ${{matrix.build_variant}}
            TSAN_OPTIONS: "halt_on_error=1"

        if: github.actor != 'restyled-io[bot]' && false
        runs-on: macos-13

        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Setup Environment
              # coreutils for stdbuf
              run: brew install coreutils
            - name:
                  Try to ensure the directories for core dumping and diagnostic
                  log collection exist and we can write them.
              run: |
                  sudo chown ${USER} /cores || true
                  mkdir -p ~/Library/Logs/DiagnosticReports || true
                  mkdir objdir-clone || true
            - name: Checkout submodules & Bootstrap
              uses: ./.github/actions/checkout-submodules-and-bootstrap
              with:
                platform: darwin
                bootstrap-log-name: bootstrap-logs-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }}

            - name: Build Python REPL and example apps
              run: |
                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv'
                  ./scripts/run_in_build_env.sh \
                   "./scripts/build/build_examples.py \
                      --target darwin-x64-all-clusters-${BUILD_VARIANT}-test \
                      build \
                      --copy-artifacts-to objdir-clone \
                   "
            - name: Run Tests
              run: |
                  scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/darwin-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factory-reset --quiet --app-args "--discriminator 3840 --interface-id -1" --script-args "-t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"'
            - name: Uploading core files
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-core-darwin-python-repl
                  path: /cores/
                  # Cores are big; don't hold on to them too long.
                  retention-days: 5
            - name: Uploading traces on failure
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: trace-data-python-repl
                  path: out/trace_data/
                  retention-days: 5
            - name: Uploading diagnostic logs
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-log-darwin-python-repl
                  path: ~/Library/Logs/DiagnosticReports/
            - name: Uploading objdir for debugging
              uses: actions/upload-artifact@v4
              if: ${{ failure() && !env.ACT }}
              with:
                  name: crash-objdir-darwin-python-repl
                  path: objdir-clone/
                  # objdirs are big; don't hold on to them too long.
                  retention-days: 5