Skip to content

Commit 13de723

Browse files
Support running test suites via a python script (project-chip#11996)
* Start adding a file that is able to name possible tests * Move test logic into separate python module, for easier code logic split * Add support for linux netns in tests including a validation shell. tested that this works in host. Updated vscode container to also allow running these * Add milliseconds for logging, add sleep for tentative ipv6 addresses * Restyle fixes * Prepare for test execution * more work to make runners functional * This can now run unit tests (at least in linux) * Restyle fixes * Add TV app as a supported application on host builds * Finished reporting on error only * Restyle and run new tests on linux using the new method * Fix the test yaml file * Remove run tv tests step - now all tests are run in one * Remove todo - it is done * Fix name of the subcommand * Run tests on darwin as well using the updated python scripts * Add glob support for running tests (hackish implmementation though through symlinks) and avoid tv tests for darwin * Add missing link file * Fix target skip glob argument for darwin * Minor updates from darwin testing: start thread after full constructor run, cleanup code a bit * Add some locking th definition logging captures * Use PTY since that removes buffering on darwin * Only use pty on darwin for IPC * Restyle fixes * Update scripts/tests/chiptest/linux.py Co-authored-by: Victor Morales <chipahuac@hotmail.com> * Update scripts/tests/chiptest/runner.py Co-authored-by: Victor Morales <chipahuac@hotmail.com> * Restyle fixes * Run 2 iterations of test suites to preserve previous behaviour, remove test_suites.sh * Add extra validation: if chip-all-clusters-app fails, make the test runner fail the test Co-authored-by: Victor Morales <chipahuac@hotmail.com>
1 parent 23fad89 commit 13de723

File tree

13 files changed

+678
-280
lines changed

13 files changed

+678
-280
lines changed

.devcontainer/devcontainer.json

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"--security-opt",
66
"seccomp=unconfined",
77
"--network=host",
8+
"--privileged",
89
"-v",
910
"/dev/bus/usb:/dev/bus/usb:ro",
1011
"--device-cgroup-rule=a 189:* rmw",

.github/workflows/tests.yaml

+19-16
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,15 @@ jobs:
8484
# actually succeeded, because that just wastes space.
8585
rsync -a out/debug/standalone/ objdir-clone || true
8686
- name: Run Tests
87-
timeout-minutes: 20
87+
timeout-minutes: 30
8888
run: |
89-
scripts/tests/test_suites.sh -n
90-
- name: Run TV Tests
91-
timeout-minutes: 10
92-
run: |
93-
scripts/tests/test_suites.sh -n -a tv
89+
./scripts/run_in_build_env.sh \
90+
"./scripts/tests/run_test_suite.py run \
91+
--iterations 2 \
92+
--chip-tool ./out/debug/standalone/chip-tool \
93+
--all-clusters-app ./out/debug/standalone/chip-all-clusters-app \
94+
--tv-app ./out/debug/standalone/chip-tv-app \
95+
"
9496
- name: Uploading core files
9597
uses: actions/upload-artifact@v2
9698
if: ${{ failure() }} && ${{ !env.ACT }}
@@ -167,17 +169,18 @@ jobs:
167169
# The idea is to not upload our objdir unless builds have
168170
# actually succeeded, because that just wastes space.
169171
rsync -a out/debug/standalone/ objdir-clone || true
170-
- name: Run Test Suites
171-
timeout-minutes: 35
172+
- name: Run Tests
173+
timeout-minutes: 45
172174
run: |
173-
scripts/tests/test_suites.sh
174-
- name: Uploading application logs
175-
uses: actions/upload-artifact@v2
176-
if: ${{ failure() }} && ${{ !env.ACT }}
177-
with:
178-
name: test-suite-app-logs-${{ matrix.type }}-${{ matrix.eventloop }}
179-
path: /tmp/test_suites_app_logs/
180-
retention-days: 5
175+
./scripts/run_in_build_env.sh \
176+
"./scripts/tests/run_test_suite.py \
177+
--target-skip-glob 'tv-*' \
178+
run \
179+
--iterations 2 \
180+
--chip-tool ./out/debug/standalone/chip-tool \
181+
--all-clusters-app ./out/debug/standalone/chip-all-clusters-app \
182+
--tv-app ./out/debug/standalone/chip-tv-app \
183+
"
181184
- name: Uploading core files
182185
uses: actions/upload-artifact@v2
183186
if: ${{ failure() }} && ${{ !env.ACT }}

.restyled.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ exclude:
6767
- "third_party/nanopb/repo/**/*"
6868
- "src/android/CHIPTool/gradlew" # gradle wrapper generated file
6969
- "third_party/android_deps/gradlew" # gradle wrapper generated file
70-
- "scripts/tests/test_suites.sh" # overly agressive shell harden
7170

7271

7372
changed_paths:

scripts/build/build/targets.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,11 @@ def HostTargets():
106106

107107
app_targets = []
108108

109-
# RPC console compilation only for native
109+
# Don't cross compile some builds
110110
app_targets.append(
111111
targets[0].Extend('rpc-console', app=HostApp.RPC_CONSOLE))
112+
app_targets.append(
113+
targets[0].Extend('tv-app', app=HostApp.TV_APP))
112114

113115
for target in targets:
114116
app_targets.append(target.Extend(

scripts/build/builders/host.py

+6
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ class HostApp(Enum):
2626
THERMOSTAT = auto()
2727
RPC_CONSOLE = auto()
2828
MIN_MDNS = auto()
29+
TV_APP = auto()
2930

3031
def ExamplePath(self):
3132
if self == HostApp.ALL_CLUSTERS:
@@ -38,6 +39,8 @@ def ExamplePath(self):
3839
return 'common/pigweed/rpc_console'
3940
if self == HostApp.MIN_MDNS:
4041
return 'minimal-mdns'
42+
if self == HostApp.TV_APP:
43+
return 'tv-app/linux'
4144
else:
4245
raise Exception('Unknown app type: %r' % self)
4346

@@ -60,6 +63,9 @@ def OutputNames(self):
6063
yield 'minimal-mdns-client.map'
6164
yield 'minimal-mdns-server'
6265
yield 'minimal-mdns-server.map'
66+
elif self == HostApp.TV_APP:
67+
yield 'chip-tv-app'
68+
yield 'chip-tv-app.map'
6369
else:
6470
raise Exception('Unknown app type: %r' % self)
6571

scripts/build/testdata/build_linux_on_x64.txt

+12
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@ gn gen --check --fail-on-unused-args --root={root}/examples/thermostat/linux {ou
6868
# Generating linux-x64-thermostat-ipv6only
6969
gn gen --check --fail-on-unused-args --root={root}/examples/thermostat/linux --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-thermostat-ipv6only
7070

71+
# Generating linux-x64-tv-app
72+
gn gen --check --fail-on-unused-args --root={root}/examples/tv-app/linux {out}/linux-x64-tv-app
73+
74+
# Generating linux-x64-tv-app-ipv6only
75+
gn gen --check --fail-on-unused-args --root={root}/examples/tv-app/linux --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-tv-app-ipv6only
76+
7177
# Building linux-arm64-all-clusters
7278
ninja -C {out}/linux-arm64-all-clusters
7379

@@ -118,3 +124,9 @@ ninja -C {out}/linux-x64-thermostat
118124

119125
# Building linux-x64-thermostat-ipv6only
120126
ninja -C {out}/linux-x64-thermostat-ipv6only
127+
128+
# Building linux-x64-tv-app
129+
ninja -C {out}/linux-x64-tv-app
130+
131+
# Building linux-x64-tv-app-ipv6only
132+
ninja -C {out}/linux-x64-tv-app-ipv6only

scripts/tests/chiptest/__init__.py

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#
2+
# Copyright (c) 2021 Project CHIP Authors
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
from pathlib import Path
18+
import os
19+
import logging
20+
21+
import chiptest.linux
22+
import chiptest.runner
23+
24+
from .test_definition import TestTarget, TestDefinition, ApplicationPaths
25+
26+
27+
def AllTests(root: str):
28+
"""Gets all the tests that can be found in the ROOT directory based on
29+
yaml file names.
30+
"""
31+
for path in Path(os.path.join(root, 'src', 'app', 'tests', 'suites')).rglob("*.yaml"):
32+
logging.debug('Found YAML: %s' % path)
33+
34+
# grab the name without the extension
35+
name = path.stem.lower()
36+
37+
if 'simulated' in name:
38+
continue
39+
40+
if name.startswith('tv_'):
41+
target = TestTarget.TV
42+
name = 'tv-' + name[3:]
43+
elif name.startswith('test_'):
44+
target = TestTarget.ALL_CLUSTERS
45+
name = 'app-' + name[5:]
46+
else:
47+
continue
48+
49+
yield TestDefinition(yaml_file=path, run_name=path.stem, name=name, target=target)
50+
51+
52+
__all__ = ['TestTarget', 'TestDefinition', 'AllTests', 'ApplicationPaths']
+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../build/glob_matcher.py

scripts/tests/chiptest/linux.py

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#
2+
# Copyright (c) 2021 Project CHIP Authors
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
"""
18+
Handles linux-specific functionality for running test cases
19+
"""
20+
21+
import logging
22+
import os
23+
import subprocess
24+
import sys
25+
import time
26+
27+
from .test_definition import ApplicationPaths
28+
29+
test_environ = os.environ.copy()
30+
31+
32+
def EnsureNetworkNamespaceAvailability():
33+
if os.getuid() == 0:
34+
logging.debug("Current user is root")
35+
logging.warn("Running as root and this will change global namespaces.")
36+
return
37+
38+
os.execvpe("unshare", ["unshare", "--map-root-user", "-n", "-m", "python3",
39+
sys.argv[0], '--internal-inside-unshare'] + sys.argv[1:], test_environ)
40+
41+
42+
def EnsurePrivateState():
43+
logging.info("Ensuring /run is privately accessible")
44+
45+
logging.debug("Making / private")
46+
if os.system("mount --make-private /") != 0:
47+
logging.error("Failed to make / private")
48+
logging.error("Are you using --privileged if running in docker?")
49+
sys.exit(1)
50+
51+
logging.debug("Remounting /run")
52+
if os.system("mount -t tmpfs tmpfs /run") != 0:
53+
logging.error("Failed to mount /run as a temporary filesystem")
54+
logging.error("Are you using --privileged if running in docker?")
55+
sys.exit(1)
56+
57+
58+
def CreateNamespacesForAppTest():
59+
"""
60+
Creates appropriate namespaces for a tool and app binaries in a simulated
61+
isolated network.
62+
"""
63+
COMMANDS = [
64+
# 2 virtual hosts: for app and for the tool
65+
"ip netns add app",
66+
"ip netns add tool",
67+
68+
# create links for switch to net connections
69+
"ip link add eth-app type veth peer name eth-app-switch",
70+
"ip link add eth-tool type veth peer name eth-tool-switch",
71+
72+
# link the connections together
73+
"ip link set eth-app netns app",
74+
"ip link set eth-tool netns tool",
75+
76+
"ip link add name br1 type bridge",
77+
"ip link set br1 up",
78+
"ip link set eth-app-switch master br1",
79+
"ip link set eth-tool-switch master br1",
80+
81+
# mark connections up
82+
"ip netns exec app ip addr add 10.10.10.1/24 dev eth-app",
83+
"ip netns exec app ip link set dev eth-app up",
84+
"ip netns exec app ip link set dev lo up",
85+
"ip link set dev eth-app-switch up",
86+
87+
"ip netns exec tool ip addr add 10.10.10.2/24 dev eth-tool",
88+
"ip netns exec tool ip link set dev eth-tool up",
89+
"ip netns exec tool ip link set dev lo up",
90+
"ip link set dev eth-tool-switch up",
91+
92+
# Force IPv6 to use ULAs that we control
93+
"ip netns exec tool ip -6 addr flush eth-tool",
94+
"ip netns exec app ip -6 addr flush eth-app",
95+
"ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool",
96+
"ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app",
97+
]
98+
99+
for command in COMMANDS:
100+
logging.debug("Executing '%s'" % command)
101+
if os.system(command) != 0:
102+
logging.error("Failed to execute '%s'" % command)
103+
logging.error("Are you using --privileged if running in docker?")
104+
sys.exit(1)
105+
106+
# IPv6 does Duplicate Address Detection even though
107+
# we know ULAs provided are isolated. Wait for 'tenative' address to be gone
108+
109+
logging.info('Waiting for IPv6 DaD to complete (no tentative addresses)')
110+
for i in range(100): # wait at most 10 seconds
111+
output = subprocess.check_output(['ip', 'addr'])
112+
if b'tentative' not in output:
113+
logging.info('No more tentative addresses')
114+
break
115+
time.sleep(0.1)
116+
else:
117+
logging.warn("Some addresses look to still be tentative")
118+
119+
120+
def PrepareNamespacesForTestExecution(in_unshare: bool):
121+
if not in_unshare:
122+
EnsureNetworkNamespaceAvailability()
123+
elif in_unshare:
124+
EnsurePrivateState()
125+
126+
CreateNamespacesForAppTest()
127+
128+
129+
def PathsWithNetworkNamespaces(paths: ApplicationPaths) -> ApplicationPaths:
130+
"""
131+
Returns a copy of paths with updated command arrays to invoke the
132+
commands in an appropriate network namespace.
133+
"""
134+
return ApplicationPaths(
135+
chip_tool='ip netns exec tool'.split() + paths.chip_tool,
136+
all_clusters_app='ip netns exec app'.split() + paths.all_clusters_app,
137+
tv_app='ip netns exec app'.split() + paths.tv_app,
138+
)

0 commit comments

Comments
 (0)