Make the build_coverage script more flexible (#37415)

yufengwangca · restyled-commits · web-flow · commit a998bf678ef9 · 2025-02-06T05:38:23.000Z
* Make the build_coverage script more flexible

* Restyled by prettier-markdown

---------

Co-authored-by: Restyled.io &lt;commits@restyled.io&gt;
diff --git a/docs/guides/BUILDING.md b/docs/guides/BUILDING.md
@@ -589,30 +589,77 @@ SDK source code has been executed. It also provides information on how often the
 Matter SDK executes segments of the code and produces a copy of the source file,
 annotated with execution frequencies.
 
-Run the following command to initiate the script:
+### How to Run
+
+```
+./scripts/build_coverage.sh [OPTIONS]
+```
+
+By default, the script
+
+Builds the Matter SDK with coverage instrumentation (unless you specify a custom
+--output_root). Runs the unit tests to generate coverage data. Produces an HTML
+coverage report located at:
+
+```
+out/coverage/coverage/html/index.html
+```
+
+You can extend the coverage scope and test types with the following options:
+
+Option Description -c, --code=<scope> Specify the scope to collect coverage
+data. - core (default): Coverage from the core Matter SDK stack - clusters:
+Coverage from cluster implementations - all: Coverage from the entire Matter SDK
+
+--yaml Also run YAML-based tests, in addition to unit tests.
+
+--python Also run Python-based tests, in addition to unit tests.
+
+-o, --output_root=DIR If specified, skip the build phase and only run coverage
+on the provided build output directory. This directory must have been built with
+use_coverage=true and have had tests run already.
+
+--target=<testname> When running unit tests, specifies a particular test target
+to run (e.g., TestEmberAttributeBuffer.run).
+
+-h, --help Print script usage and exit.
+
+### Examples
+
+Run coverage with the default scope (core) and only unit tests:
 
 ```
 ./scripts/build_coverage.sh
 ```
 
-By default, the code coverage script is performed at the unit testing level.
-Unit tests are created by developers, thus giving them the best overview of what
-tests to include in unit testing. You can extend the coverage test by scope and
-ways of execution with the following parameters:
+Run coverage including YAML tests (plus the always-enabled unit tests):
+
+```
+./scripts/build_coverage.sh --yaml
+```
+
+Run coverage including Python tests (plus the always-enabled unit tests):
 
 ```
-  -c, --code                Specify which scope to collect coverage data.
-                            'core': collect coverage data from core stack in Matter SDK. --default
-                            'clusters': collect coverage data from clusters implementation in Matter SDK.
-                            'all': collect coverage data from Matter SDK.
-  -t, --tests               Specify which tools to run the coverage check.
-                            'unit': Run unit test to drive the coverage check. --default
-                            'yaml': Run yaml test to drive the coverage check.
-                            'all': Run unit & yaml test to drive the coverage check.
+./scripts/build_coverage.sh --python
 ```
 
-Also, see the up-to-date unit testing coverage report of the Matter SDK
-(collected daily) at:
+Run coverage including both YAML and Python tests:
+
+```
+./scripts/build_coverage.sh --yaml --python
+```
+
+Change coverage scope to all (core + clusters) and run YAML tests:
+
+```
+./scripts/build_coverage.sh --code=all --yaml
+```
+
+### Viewing Coverage Results
+
+After the script completes, open the following file in your web browser to view
+the HTML coverage report:
 [matter coverage](https://matter-build-automation.ue.r.appspot.com).
 
 ## Maintaining Matter
diff --git a/scripts/build_coverage.sh b/scripts/build_coverage.sh
@@ -46,37 +46,45 @@ CHIP_ROOT=$(_normpath "$(dirname "$0")/..")
 OUTPUT_ROOT="$CHIP_ROOT/out/coverage"
 COVERAGE_ROOT="$OUTPUT_ROOT/coverage"
 SUPPORTED_CODE=(core clusters all)
-SUPPORTED_TESTS=(unit yaml all)
 CODE="core"
-TESTS="unit"
+
 skip_gn=false
 TEST_TARGET=check
 
-help() {
+# By default, do not run YAML or Python tests
+ENABLE_YAML=false
+ENABLE_PYTHON=false
 
-    echo "Usage: $file_name [--output_root=<output_root>] [--code=<core|clusters|all>] [--tests=<unit|yaml|all>]"
+help() {
+    echo "Usage: $file_name [--output_root=<output_root>] [--code=<core|clusters|all>] [Test options"
+    echo
+    echo "Misc:"
+    echo "    -h, --help              Print this help, then exit."
+    echo
+    echo "Build/Output options:"
+    echo "    -o, --output_root=DIR   Set the build output directory."
+    echo "                            When set manually, script only runs lcov on the provided build output."
+    echo "                            This directory must be built with 'use_coverage=true' and 'ninja check' must have run."
+    echo
+    echo "    -c, --code=TYPE         Specify which scope to collect coverage data. One of:"
+    echo "        core      - (default) coverage from core stack in Matter SDK."
+    echo "        clusters  - coverage from cluster implementations in Matter SDK."
+    echo "        all       - coverage from entire Matter SDK."
     echo
-    echo "Misc:
-  -h, --help                Print this help, then exit."
+    echo "Test options:"
+    echo "    --yaml                  In addition to unit tests, run YAML-based tests."
+    echo "    --python                In addition to unit tests, run Python-based tests."
+    echo "                            Both can be combined if needed."
+    echo
+    echo "    --target=TARGET         Specific test target to run for unit tests (e.g. 'TestEmberAttributeBuffer.run')."
     echo
-    echo "Options:
-  -o, --output_root         Set the build output directory.  When set manually, performs only lcov stage
-                            on provided build output.  Assumes output_root has been built with 'use_coverage=true'
-                            and that 'ninja check' was run.
-  -c, --code                Specify which scope to collect coverage data.
-                            'core': collect coverage data from core stack in Matter SDK. --default
-                            'clusters': collect coverage data from clusters implementation in Matter SDK.
-                            'all': collect coverage data from Matter SDK.
-  -t, --tests               Specify which tools to run the coverage check.
-                            'unit': Run unit test to drive the coverage check. --default
-                            'yaml': Run yaml test to drive the coverage check.
-                            'all': Run unit & yaml test to drive the coverage check.
-  --target                  Specific test target to run (e.g. TestEmberAttributeBuffer.run)
-  "
 }
 
 file_name=${0##*/}
 
+# ------------------------------------------------------------------------------
+# Parse arguments
+# ------------------------------------------------------------------------------
 for i in "$@"; do
     case $i in
         -h | --help)
@@ -87,19 +95,24 @@ for i in "$@"; do
             CODE="${i#*=}"
             shift
             ;;
-        -t=* | --tests=*)
-            TESTS="${i#*=}"
-            shift
-            ;;
         --target=*)
             TEST_TARGET="${i#*=}"
+            shift
             ;;
         -o=* | --output_root=*)
             OUTPUT_ROOT="${i#*=}"
             COVERAGE_ROOT="$OUTPUT_ROOT/coverage"
             skip_gn=true
             shift
             ;;
+        --yaml)
+            ENABLE_YAML=true
+            shift
+            ;;
+        --python)
+            ENABLE_PYTHON=true
+            shift
+            ;;
         *)
             echo "Unknown Option \"$1\""
             echo
@@ -109,49 +122,63 @@ for i in "$@"; do
     esac
 done
 
+# Validate code argument
 if [[ ! " ${SUPPORTED_CODE[@]} " =~ " ${CODE} " ]]; then
     echo "ERROR: Code $CODE not supported"
     exit 1
 fi
 
-if [[ ! " ${SUPPORTED_TESTS[@]} " =~ " ${TESTS} " ]]; then
-    echo "ERROR: Tests $TESTS not supported"
-    exit 1
-fi
-
+# ------------------------------------------------------------------------------
+# Build & Test
+# ------------------------------------------------------------------------------
 if [ "$skip_gn" == false ]; then
-    # Ensure we have a compilation environment
+    # Ensure environment is set
     source "$CHIP_ROOT/scripts/activate.sh"
 
-    # Generates ninja files
+    # Generate ninja files
     EXTRA_GN_ARGS=""
-    if [[ "$TESTS" == "yaml" || "$TESTS" == "all" ]]; then
+
+    # We only need 'chip_build_all_clusters_app' if we run YAML tests
+    if [ "$ENABLE_YAML" == true ]; then
         EXTRA_GN_ARGS="$EXTRA_GN_ARGS chip_build_all_clusters_app=true"
     else
+        # Otherwise skip building tools
         EXTRA_GN_ARGS="$EXTRA_GN_ARGS chip_build_tools=false"
     fi
-    gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="use_coverage=true$EXTRA_GN_ARGS"
 
-    # Run unit tests
-    if [[ "$TESTS" == "unit" || "$TESTS" == "all" ]]; then
-        ninja -C "$OUTPUT_ROOT" "$TEST_TARGET"
-    fi
+    gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="use_coverage=true $EXTRA_GN_ARGS"
+
+    #
+    # 1) Always run unit tests
+    #
+    ninja -C "$OUTPUT_ROOT" "$TEST_TARGET"
 
-    # Run yaml tests
-    if [[ "$TESTS" == "yaml" || "$TESTS" == "all" ]]; then
+    #
+    # 2) Run YAML tests if requested
+    #
+    if [ "$ENABLE_YAML" == true ]; then
         ninja -C "$OUTPUT_ROOT"
 
         scripts/run_in_build_env.sh \
             "./scripts/tests/run_test_suite.py \
-             --chip-tool ""$OUTPUT_ROOT/chip-tool \
+             --chip-tool \"$OUTPUT_ROOT/chip-tool\" \
              run \
              --iterations 1 \
              --test-timeout-seconds 120 \
-             --all-clusters-app ""$OUTPUT_ROOT/chip-all-clusters-app
-          "
+             --all-clusters-app \"$OUTPUT_ROOT/chip-all-clusters-app\""
+    fi
+
+    #
+    # 3) Run Python tests if requested
+    #
+    if [ "$ENABLE_PYTHON" == true ]; then
+        echo "Running Python tests ..."
+        # TODO: run python tests.
     fi
 
-    # Remove misc support components from coverage statistics
+    # ----------------------------------------------------------------------------
+    # Remove objects we do NOT want included in coverage
+    # ----------------------------------------------------------------------------
     rm -rf "$OUTPUT_ROOT/obj/src/app/app-platform"
     rm -rf "$OUTPUT_ROOT/obj/src/app/common"
     rm -rf "$OUTPUT_ROOT/obj/src/app/util/mock"
@@ -162,24 +189,45 @@ if [ "$skip_gn" == false ]; then
     rm -rf "$OUTPUT_ROOT/obj/src/platform"
     rm -rf "$OUTPUT_ROOT/obj/src/tools"
 
-    # Remove unit test itself from coverage statistics
+    # Remove unit test objects from coverage
     find "$OUTPUT_ROOT/obj/src/" -depth -name 'tests' -exec rm -rf {} \;
 
+    # Restrict coverage to 'core' or 'clusters' if specified
     if [ "$CODE" == "core" ]; then
         rm -rf "$OUTPUT_ROOT/obj/src/app/clusters"
     elif [ "$CODE" == "clusters" ]; then
         mv "$OUTPUT_ROOT/obj/src/app/clusters" "$OUTPUT_ROOT/obj/clusters"
         rm -rf "$OUTPUT_ROOT/obj/src"
-        mkdir "$OUTPUT_ROOT/obj/src"
+        mkdir -p "$OUTPUT_ROOT/obj/src"
         mv "$OUTPUT_ROOT/obj/clusters" "$OUTPUT_ROOT/obj/src/clusters"
     fi
 fi
 
+# ------------------------------------------------------------------------------
+# Coverage Generation
+# ------------------------------------------------------------------------------
 mkdir -p "$COVERAGE_ROOT"
-lcov --initial --capture --ignore-errors inconsistent --directory "$OUTPUT_ROOT/obj/src" --exclude="$PWD"/zzz_generated/* --exclude="$PWD"/third_party/* --exclude=/usr/include/* --output-file "$COVERAGE_ROOT/lcov_base.info"
-lcov --capture --ignore-errors inconsistent --directory "$OUTPUT_ROOT/obj/src" --exclude="$PWD"/zzz_generated/* --exclude="$PWD"/third_party/* --exclude=/usr/include/* --output-file "$COVERAGE_ROOT/lcov_test.info"
-lcov --ignore-errors inconsistent --add-tracefile "$COVERAGE_ROOT/lcov_base.info" --add-tracefile "$COVERAGE_ROOT/lcov_test.info" --output-file "$COVERAGE_ROOT/lcov_final.info"
-genhtml "$COVERAGE_ROOT/lcov_final.info" --output-directory "$COVERAGE_ROOT/html" --title "SHA:$(git rev-parse HEAD)" --header-title "Matter SDK Coverage Report"
 
-# Copy webapp's YAML file to the coverage output directory
-cp "$CHIP_ROOT/integrations/appengine/webapp_config.yaml" "$COVERAGE_ROOT/webapp_config.yaml"
+lcov --initial --capture --directory "$OUTPUT_ROOT/obj/src" \
+    --exclude="$PWD"/zzz_generated/* \
+    --exclude="$PWD"/third_party/* \
+    --exclude=/usr/include/* \
+    --output-file "$COVERAGE_ROOT/lcov_base.info"
+
+lcov --capture --directory "$OUTPUT_ROOT/obj/src" \
+    --exclude="$PWD"/zzz_generated/* \
+    --exclude="$PWD"/third_party/* \
+    --exclude=/usr/include/* \
+    --output-file "$COVERAGE_ROOT/lcov_test.info"
+
+lcov --add-tracefile "$COVERAGE_ROOT/lcov_base.info" \
+    --add-tracefile "$COVERAGE_ROOT/lcov_test.info" \
+    --output-file "$COVERAGE_ROOT/lcov_final.info"
+
+genhtml "$COVERAGE_ROOT/lcov_final.info" \
+    --output-directory "$COVERAGE_ROOT/html" \
+    --title "SHA:$(git rev-parse HEAD)" \
+    --header-title "Matter SDK Coverage Report"
+
+cp "$CHIP_ROOT/integrations/appengine/webapp_config.yaml" \
+    "$COVERAGE_ROOT/webapp_config.yaml"
