Make main build python env not depend on all platform build requirements. (#27267)

* Attempt to separate out requirements, so that bootstrap/activate can selectively add more things

* A few more requirements updates

* Also update standalone build requirements

* Add some intermediate requirements text files

* Updated more references to requirements.txt

* Preserve environment root

* Fix memory txt, make platform bootstrap configurable

* Make sure constraints are being used for pip install requirements for various platforms

* Fix activate not installing additional stuff unless asked for

* Restyled by gn

* Restyled by prettier-json

* Restyled by shellharden

* Restyled by shfmt

* Update comments

* Preserve CIPD_CACHE_DIR ...we may want to cache this instead of the entire .environment at some point

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>
Co-authored-by: Restyled.io <commits@restyled.io>

Author: 3 people

.gn

@@ -27,7 +27,7 @@ default_args = {
   pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"
 
   pw_build_PIP_CONSTRAINTS = [ "//scripts/setup/constraints.txt" ]
-  pw_build_PIP_REQUIREMENTS = [ "//scripts/setup/requirements.txt" ]
+  pw_build_PIP_REQUIREMENTS = [ "//scripts/setup/requirements.build.txt" ]
 
   # GN target to use for the default Python build venv.
   pw_build_PYTHON_BUILD_VENV = "//:matter_build_venv"

config/esp32/.gn

@@ -26,8 +26,10 @@ default_args = {
 
   pw_build_PIP_CONSTRAINTS =
       [ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ]
-  pw_build_PIP_REQUIREMENTS =
-      [ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ]
+  pw_build_PIP_REQUIREMENTS = [
+    "//third_party/connectedhomeip/scripts/setup/requirements.build.txt",
+    "//third_party/connectedhomeip/scripts/setup/requirements.esp32.txt",
+  ]
 
   import("//args.gni")
 }

config/standalone/args.gni

@@ -19,4 +19,5 @@ import("//build_overrides/chip.gni")
 chip_build_tests = false
 
 pw_build_PIP_CONSTRAINTS = [ "${chip_root}/scripts/setup/constraints.txt" ]
-pw_build_PIP_REQUIREMENTS = [ "${chip_root}/scripts/setup/requirements.txt" ]
+pw_build_PIP_REQUIREMENTS =
+    [ "${chip_root}/scripts/setup/requirements.build.txt" ]

examples/common/pigweed/rpc_console/.gn

@@ -23,7 +23,7 @@ default_args = {
   pw_build_PIP_CONSTRAINTS =
       [ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ]
   pw_build_PIP_REQUIREMENTS =
-      [ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ]
+      [ "//third_party/connectedhomeip/scripts/setup/requirements.build.txt" ]
 
   pw_build_USE_NEW_PYTHON_BUILD = true
 }

scripts/setup/bootstrap.sh

@@ -14,6 +14,38 @@
 # limitations under the License.
 #
 
+_install_additional_pip_requirements() {
+    _SETUP_PLATFORM=$1
+    shift
+
+    # figure out additional pip install items
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            -p | --platform)
+                _SETUP_PLATFORM=$2
+                shift # argument
+                shift # value
+                ;;
+            *)
+                shift
+                ;;
+        esac
+    done
+
+    if ! [ -z "$_SETUP_PLATFORM" ]; then
+        IFS="," read -r -a _PLATFORMS <<<"$_SETUP_PLATFORM"
+        for platform in "${_PLATFORMS[@]}"; do
+            # Allow none as an alias of nothing extra installed (like -p none)
+            if [ "$platform" != "none" ]; then
+                echo "Installing pip requirements for $platform..."
+                pip install -q \
+                    -r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \
+                    -c "$_CHIP_ROOT/scripts/setup/constraints.txt"
+            fi
+        done
+    fi
+}
+
 _bootstrap_or_activate() {
     if [ -n "$BASH" ]; then
         local _BOOTSTRAP_PATH="${BASH_SOURCE[0]}"
@@ -24,7 +56,7 @@ _bootstrap_or_activate() {
     local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}"
     local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}"
     # Strip off the 'scripts[/setup]' directory, leaving the root of the repo.
-    local _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." && pwd)"
+    _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." && pwd)"
 
     local _CONFIG_FILE="scripts/setup/environment.json"
 
@@ -91,21 +123,42 @@ EOF
             --config-file "$_CHIP_ROOT/$_CONFIG_FILE" \
             --virtualenv-gn-out-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT/gn_out"
         pw_finalize bootstrap "$_SETUP_SH"
+        _ACTION_TAKEN="bootstrap"
     else
         pw_activate
         pw_finalize activate "$_SETUP_SH"
+        _ACTION_TAKEN="activate"
     fi
 }
 
+# remember PW_ENVIRONMENT_ROOT so that things like another
+# bootstrap or run_in_build_env.sh can be executed in a build env
+_ORIGINAL_PW_ENVIRONMENT_ROOT="$PW_ENVIRONMENT_ROOT"
+
 _bootstrap_or_activate "$0"
+
+if [ "$_ACTION_TAKEN" = "bootstrap" ]; then
+    # By default, install all extra pip dependencies even if slow. -p/--platform
+    # arguments may override this default.
+    _install_additional_pip_requirements "all" "$@"
+else
+    _install_additional_pip_requirements "none" "$@"
+fi
+
 unset -f _bootstrap_or_activate
+unset -f _install_additional_pip_requirements
 
 pw_cleanup
 
+unset _ACTION_TAKEN
+unset _CHIP_ROOT
 unset PW_CIPD_INSTALL_DIR
-unset CIPD_CACHE_DIR
 unset _PW_BANNER_FUNC
 unset _PW_TEXT
 unset PW_DOCTOR_SKIP_CIPD_CHECKS
 
 unset -f _chip_bootstrap_banner
+
+if ! [ -z "$_ORIGINAL_PW_ENVIRONMENT_ROOT" ]; then
+    export PW_ENVIRONMENT_ROOT="$_ORIGINAL_PW_ENVIRONMENT_ROOT"
+fi

scripts/setup/constraints.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=constraints.txt --resolver=backtracking --strip-extras requirements.txt
+#    pip-compile --output-file=constraints.txt --resolver=backtracking --strip-extras requirements.all.txt
 #
 anytree==2.8.0
     # via -r requirements.txt

scripts/setup/environment.json

@@ -8,7 +8,7 @@
     "virtualenv": {
         "gn_root": ".",
         "gn_targets": [":python_packages.install"],
-        "requirements": ["scripts/setup/requirements.txt"],
+        "requirements": ["scripts/setup/requirements.build.txt"],
         "constraints": ["scripts/setup/constraints.txt"],
         "gn_args": ["chip_crypto=\"boringssl\""]
     },

scripts/setup/environment_no_cipd.json

@@ -2,7 +2,7 @@
     "virtualenv": {
         "gn_root": ".",
         "gn_targets": [":python_packages.install"],
-        "requirements": ["scripts/setup/requirements.txt"],
+        "requirements": ["scripts/setup/requirements.build.txt"],
         "constraints": ["scripts/setup/constraints.txt"],
         "gn_args": ["chip_crypto=\"boringssl\""]
     },

scripts/setup/requirements.txt scripts/setup/requirements.all.txt

@@ -8,26 +8,16 @@ virtualenv
 -c constraints.esp32.txt
 -r requirements.esp32.txt
 
-# mbed-os
 -r requirements.mbed.txt
-
-# bouffalolab
 -r requirements.bouffalolab.txt
-
-# openiotsdk
 -r requirements.openiotsdk.txt
-
-# Infineon
 -r requirements.infineon.txt
-
-# TI
 -r requirements.ti.txt
-
-# Telink
 -r requirements.telink.txt
-
-# cirque tests
-requests>=2.24.0
+-r requirements.zephyr.txt
+-r requirements.cirque.txt
+-r requirements.memory.txt
+-r requirements.yaml_tests.txt
 
 # device controller wheel package
 wheel; sys_platform == 'linux'
@@ -40,17 +30,6 @@ pyobjc-framework-corebluetooth; sys_platform == 'darwin'
 portpicker
 mobly
 
-# zephyr
-west>=0.12.0
-intelhex
-cbor
-cbor2
-jsonschema
-
-# happy tests
-lockfile
-psutil >= 5.7.3
-
 # pigweed
 ipython
 appnope
@@ -63,13 +42,6 @@ mypy-protobuf==3.2.0
 protobuf==3.20.1
 types-protobuf==3.19.22
 
-# scripts/tools/memory
-anytree
-cxxfilt
-ghapi
-pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64'
-tabulate
-
 cryptography
 
 # python unit tests
@@ -78,6 +50,3 @@ colorama
 # update tornado for pw_watch
 tornado
 
-# YAML test harness
-diskcache
-websockets

scripts/setup/requirements.cirque.txt

@@ -0,0 +1 @@
+requests>=2.24.0

scripts/setup/requirements.memory.txt

@@ -0,0 +1,6 @@
+# scripts/tools/memory
+anytree
+cxxfilt
+ghapi
+pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64'
+tabulate

scripts/setup/requirements.yaml_tests.txt

@@ -0,0 +1,2 @@
+diskcache
+websockets

scripts/setup/requirements.zephyr.txt

@@ -0,0 +1,5 @@
+west>=0.12.0
+intelhex
+cbor
+cbor2
+jsonschema

scripts/tools/telink/readme.md

@@ -35,7 +35,8 @@ tool.
 
 ```shell
 cd path/to/connectedhomeip/scripts/tools/telink/
-python3 -m pip install -r requirements.txt
+python3 -m pip install -r requirements.build.txt
+python3 -m pip install -r requirements.telink.txt
 ```
 
 ## Usage

src/test_driver/efr32/.gn

@@ -27,7 +27,7 @@ default_args = {
   pw_build_PIP_CONSTRAINTS =
       [ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ]
   pw_build_PIP_REQUIREMENTS =
-      [ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ]
+      [ "//third_party/connectedhomeip/scripts/setup/requirements.build.txt" ]
 
   import("//args.gni")
 }