Skip to content

Commit 8ef99f6

Browse files
authored
Remove costly python requirements on Raspberry Pi (project-chip#7635)
It takes up to 1 hour to boostrap on Raspberry Pi, largely due to adding "pandas" which is expensive to build and doesn't have a linux aarch64 wheel on pypi. Remove packages with no wheels from the requirements on aarch64 to save time (pandas, gdbgui). This switches the compiled requirements.txt to a "constraints.txt" which is like a requirements file but only contains version constraints and does not automatically install its contents. That way, we can specify what version to use for packages on amd64 without forcing their installation on the Pi. See pip help install for more details on this feature. This reduces initial Pi 4 Python bootstrap down to ~4m from ~50m.
1 parent f034fc6 commit 8ef99f6

File tree

5 files changed

+258
-256
lines changed

5 files changed

+258
-256
lines changed

.gn

+1
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,5 @@ script_executable = "python3"
2525

2626
default_args = {
2727
pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"
28+
pw_build_PIP_CONSTRAINTS = ["//scripts/constraints.txt"]
2829
}

scripts/constraints.txt

+205
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
#
2+
# This file is autogenerated by pip-compile
3+
# To update, run:
4+
#
5+
# pip-compile --output-file=constraints.txt requirements.txt
6+
#
7+
anytree==2.8.0
8+
# via -r requirements.txt
9+
appdirs==1.4.4
10+
# via
11+
# -r requirements.txt
12+
# virtualenv
13+
appnope==0.1.2
14+
# via -r requirements.txt
15+
backcall==0.2.0
16+
# via ipython
17+
bidict==0.21.2
18+
# via python-socketio
19+
bitstring==3.1.7
20+
# via -r requirements.esp32.txt
21+
brotli==1.0.9
22+
# via flask-compress
23+
certifi==2021.5.30
24+
# via requests
25+
cffi==1.14.5
26+
# via cryptography
27+
chardet==4.0.0
28+
# via requests
29+
click==8.0.1
30+
# via
31+
# -r requirements.esp32.txt
32+
# flask
33+
# pip-tools
34+
colorama==0.4.4
35+
# via west
36+
coloredlogs==15.0.1
37+
# via -r requirements.txt
38+
cryptography==3.4.7
39+
# via -r requirements.esp32.txt
40+
cxxfilt==0.2.2
41+
# via -r requirements.txt
42+
dbus-python==1.2.16 ; sys_platform == "linux"
43+
# via -r requirements.txt
44+
decorator==5.0.9
45+
# via ipython
46+
distlib==0.3.2
47+
# via virtualenv
48+
docopt==0.6.2
49+
# via pykwalify
50+
ecdsa==0.17.0
51+
# via -r requirements.esp32.txt
52+
filelock==3.0.12
53+
# via virtualenv
54+
flask-compress==1.10.0
55+
# via gdbgui
56+
flask-socketio==2.9.6
57+
# via gdbgui
58+
flask==0.12.5
59+
# via
60+
# flask-compress
61+
# flask-socketio
62+
# gdbgui
63+
future==0.18.2
64+
# via
65+
# -r requirements.esp32.txt
66+
# mobly
67+
gdbgui==0.13.2.0 ; platform_machine != "aarch64"
68+
# via -r requirements.esp32.txt
69+
gevent==1.5.0
70+
# via gdbgui
71+
greenlet==1.1.0
72+
# via gevent
73+
humanfriendly==9.2
74+
# via coloredlogs
75+
idna==2.10
76+
# via requests
77+
intelhex==2.3.0
78+
# via -r requirements.txt
79+
ipython-genutils==0.2.0
80+
# via traitlets
81+
ipython==7.24.1
82+
# via -r requirements.txt
83+
itsdangerous==2.0.1
84+
# via flask
85+
jedi==0.18.0
86+
# via ipython
87+
jinja2==3.0.1
88+
# via flask
89+
lockfile==0.12.2
90+
# via -r requirements.txt
91+
markupsafe==2.0.1
92+
# via jinja2
93+
matplotlib-inline==0.1.2
94+
# via ipython
95+
mobly==1.10.1
96+
# via -r requirements.txt
97+
numpy==1.20.3
98+
# via pandas
99+
packaging==20.9
100+
# via west
101+
pandas==1.2.4 ; platform_machine != "aarch64"
102+
# via -r requirements.txt
103+
parso==0.8.2
104+
# via jedi
105+
pep517==0.10.0
106+
# via pip-tools
107+
pexpect==4.8.0
108+
# via ipython
109+
pgi==0.0.11.2 ; sys_platform == "linux"
110+
# via -r requirements.txt
111+
pickleshare==0.7.5
112+
# via ipython
113+
pip-tools==6.1.0
114+
# via -r requirements.txt
115+
portpicker==1.4.0
116+
# via
117+
# -r requirements.txt
118+
# mobly
119+
prompt-toolkit==3.0.18
120+
# via ipython
121+
protobuf==3.17.3
122+
# via -r requirements.txt
123+
psutil==5.8.0
124+
# via
125+
# -r requirements.txt
126+
# mobly
127+
ptyprocess==0.7.0
128+
# via pexpect
129+
pycparser==2.20
130+
# via cffi
131+
pyelftools==0.27
132+
# via -r requirements.esp32.txt
133+
pygdbmi==0.9.0.2
134+
# via
135+
# -r requirements.esp32.txt
136+
# gdbgui
137+
pygments==2.9.0
138+
# via
139+
# gdbgui
140+
# ipython
141+
pykwalify==1.8.0
142+
# via west
143+
pyparsing==2.3.1
144+
# via
145+
# -r requirements.esp32.txt
146+
# packaging
147+
pyserial==3.5
148+
# via
149+
# -r requirements.esp32.txt
150+
# mobly
151+
python-dateutil==2.8.1
152+
# via
153+
# pandas
154+
# pykwalify
155+
python-engineio==4.2.0
156+
# via python-socketio
157+
python-socketio==5.3.0
158+
# via flask-socketio
159+
pytz==2021.1
160+
# via pandas
161+
pyyaml==5.4.1
162+
# via
163+
# mobly
164+
# west
165+
reedsolo==1.5.4
166+
# via -r requirements.esp32.txt
167+
requests==2.25.1
168+
# via -r requirements.txt
169+
ruamel.yaml.clib==0.2.2
170+
# via ruamel.yaml
171+
ruamel.yaml==0.17.9
172+
# via pykwalify
173+
six==1.16.0
174+
# via
175+
# anytree
176+
# ecdsa
177+
# protobuf
178+
# python-dateutil
179+
# virtualenv
180+
timeout-decorator==0.5.0
181+
# via mobly
182+
toml==0.10.2
183+
# via pep517
184+
traitlets==5.0.5
185+
# via
186+
# ipython
187+
# matplotlib-inline
188+
urllib3==1.26.5
189+
# via requests
190+
virtualenv==20.4.7
191+
# via -r requirements.txt
192+
watchdog==2.1.2
193+
# via -r requirements.txt
194+
wcwidth==0.2.5
195+
# via prompt-toolkit
196+
werkzeug==0.16.1
197+
# via flask
198+
west==0.11.0
199+
# via -r requirements.txt
200+
wheel==0.36.2
201+
# via -r requirements.txt
202+
203+
# The following packages are considered to be unsafe in a requirements file:
204+
# pip
205+
# setuptools

scripts/requirements.esp32.txt

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
setuptools>=21
2+
click>=5.0
3+
pyserial>=3.0
4+
future>=0.15.2
5+
cryptography>=2.1.4
6+
pyparsing>=2.0.3,<2.4.0
7+
pyelftools>=0.22
8+
gdbgui==0.13.2.0 ; platform_machine != 'aarch64'
9+
pygdbmi<=0.9.0.2
10+
reedsolo>=1.5.3,<=1.5.4
11+
bitstring>=3.1.6
12+
ecdsa>=0.16.0

scripts/requirements.in

-52
This file was deleted.

0 commit comments

Comments
 (0)