Skip to content

Commit 5f4a1a9

Browse files
tests: on_target: add fota test
Add fota test. Signed-off-by: Giacomo Dematteis <giacomo.dematteis@nordicsemi.no>
1 parent d51cd0d commit 5f4a1a9

File tree

9 files changed

+534
-48
lines changed

9 files changed

+534
-48
lines changed

.github/workflows/target-test.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,8 @@ jobs:
123123
shell: bash
124124
env:
125125
SEGGER: ${{ env.RUNNER_SERIAL_NUMBER }}
126-
# IMEI: ${{ secrets.IMEI }}
127-
# FINGERPRINT: ${{ secrets.FINGERPRINT }}
126+
UUID: ${{ env.UUID }}
127+
NRFCLOUD_API_KEY: ${{ secrets.NRF_CLOUD_API_KEY }}
128128
LOG_FILENAME: att_test_log
129129
TEST_REPORT_NAME: ATT Firwmare Test Report
130130
# MEMFAULT_ORGANIZATION_TOKEN: ${{ secrets.MEMFAULT_ORGANIZATION_TOKEN }}

tests/on_target/README.md

+16-35
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22

33
## Run test locally
44

5+
NOTE: The tests have been tested on Ubuntu 22.04. For details on how to install Docker please refer to the Docker documentation https://docs.docker.com/engine/install/ubuntu/
6+
57
### Setup docker
68
```shell
79
docker pull ghcr.io/hello-nrfcloud/firmware:docker-v1.0.3
810
cd <path_to_att_dir>
9-
west build -p -b thingy91x/nrf9151/ns app
10-
cp build/merged.hex tests/on_target/artifacts/asset-tracker-template-aaa000-thingy91x-nrf91.hex
1111
docker run --rm -it \
1212
--privileged \
1313
-v /dev:/dev:rw \
@@ -16,59 +16,40 @@ docker run --rm -it \
1616
-v /opt/setup-jlink:/opt/setup-jlink \
1717
ghcr.io/hello-nrfcloud/firmware:docker-v1.0.3 \
1818
/bin/bash
19+
cd asset-tracker-template/tests/on_target
1920
```
2021

21-
NOTE: The tests have been tested on Ubuntu 22.04. For details on how to install Docker please refer to the Docker documentation https://docs.docker.com/engine/install/ubuntu/
22-
2322
### Verify nrfutil/jlink works
2423
```shell
2524
JLinkExe -V
2625
nrfutil -V
2726
```
2827

29-
### Install requirements
30-
```shell
31-
cd asset-tracker-template/tests/on_target
32-
pip install -r requirements.txt --break-system-packages
33-
```
28+
### NRF91 tests
29+
Precondition: thingy91x with segger fw on 53
3430

35-
### Get SEGGER ID
31+
Get device id
3632
```shell
3733
nrfutil device list
3834
```
3935

40-
### Run UART tests
41-
42-
Precondition: thingy91x with segger fw on 53
43-
36+
Set env
4437
```shell
4538
export SEGGER=<your_segger>
46-
pytest -s -v -m "dut1 and uart" tests
4739
```
4840

49-
### Run FOTA tests
50-
51-
Precondition: thingy91x with segger fw on 53
52-
41+
Additional fota and memfault envs
5342
```shell
54-
export SEGGER=<your_segger>
55-
export IMEI=<your_imei>
56-
export FINGERPRINT=<your_fingerprint>
57-
pytest -s -v -m "dut1 and fota" tests
43+
export UUID=<your_imei>
44+
export NRFCLOUD_API_KEY=<your_nrfcloud_api_key>
5845
```
5946

60-
### Run DFU tests
61-
62-
Precondition: thingy91x with external debugger attached
63-
64-
IMPORTANT: switch must be on nrf53.
65-
66-
Set all this bunch of env variables appropriately (see worflow for details):
67-
SEGGER_NRF53, SEGGER_NRF91, UART_ID, NRF53_HEX_FILE, NRF53_APP_UPDATE_ZIP,
68-
NRF53_BL_UPDATE_ZIP, NRF91_HEX_FILE, NRF91_APP_UPDATE_ZIP, NRF91_BL_UPDATE_ZIP
69-
70-
```
71-
pytest -s -v -m dut2 tests
47+
Run desired tests, example commands
48+
```shell
49+
pytest -s -v -m "not slow" tests
50+
pytest -s -v -m "not slow" tests/test_functional/test_uart_output.py
51+
pytest -s -v -m "not slow" tests/test_functional/test_location.py::test_wifi_location
52+
pytest -s -v -m "slow" tests/test_functional/test_fota.py::test_full_mfw_fota
7253
```
7354

7455
## Test docker image version control
-7.62 MB
Binary file not shown.
7.73 MB
Binary file not shown.
Binary file not shown.

tests/on_target/tests/conftest.py

+42-1
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,16 @@
1212
import sys
1313
sys.path.append(os.getcwd())
1414
from utils.logger import get_logger
15+
from utils.nrfcloud_fota import NRFCloudFOTA
1516

1617
logger = get_logger()
1718

1819
UART_TIMEOUT = 60 * 30
1920

2021
SEGGER = os.getenv('SEGGER')
2122
UART_ID = os.getenv('UART_ID', SEGGER)
22-
FOTADEVICE_FINGERPRINT = os.getenv('FINGERPRINT')
23+
FOTADEVICE_UUID = os.getenv('UUID')
24+
NRFCLOUD_API_KEY = os.getenv('NRFCLOUD_API_KEY')
2325

2426
def get_uarts():
2527
base_path = "/dev/serial/by-id"
@@ -67,6 +69,33 @@ def t91x_board():
6769

6870
scan_log_for_assertions(uart_log)
6971

72+
@pytest.fixture(scope="function")
73+
def t91x_fota(t91x_board):
74+
if not NRFCLOUD_API_KEY:
75+
pytest.skip("NRFCLOUD_API_KEY environment variable not set")
76+
if not FOTADEVICE_UUID:
77+
pytest.skip("UUID environment variable not set")
78+
79+
fota = NRFCloudFOTA(api_key=NRFCLOUD_API_KEY)
80+
device_id = FOTADEVICE_UUID
81+
data = {
82+
'job_id': '',
83+
'bundle_id': ''
84+
}
85+
fota.cancel_incomplete_jobs(device_id)
86+
87+
yield types.SimpleNamespace(
88+
fota=fota,
89+
uart=t91x_board.uart,
90+
device_id=device_id,
91+
data=data
92+
)
93+
94+
fota.cancel_incomplete_jobs(device_id)
95+
if data['bundle_id']:
96+
fota.delete_bundle(data['bundle_id'])
97+
98+
7099
@pytest.fixture(scope="module")
71100
def t91x_traces(t91x_board):
72101
all_uarts = get_uarts()
@@ -91,3 +120,15 @@ def hex_file():
91120
return os.path.join(artifacts_dir, file)
92121

93122
pytest.fail("No matching firmware .hex file found in the artifacts directory")
123+
124+
@pytest.fixture(scope="session")
125+
def bin_file():
126+
# Search for the firmware bin file in the artifacts folder
127+
artifacts_dir = "artifacts"
128+
hex_pattern = r"asset-tracker-template-[0-9a-z\.]+-thingy91x-nrf91-update-signed\.bin"
129+
130+
for file in os.listdir(artifacts_dir):
131+
if re.match(hex_pattern, file):
132+
return os.path.join(artifacts_dir, file)
133+
134+
pytest.fail("No matching firmware .bin file found in the artifacts directory")

tests/on_target/tests/pytest.ini

+1-10
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,3 @@
11
[pytest]
22
markers =
3-
dut1: device used for uart and fota tests
4-
dut2: device used for dfu tests
5-
uart: uart tests
6-
conn_bridge: connectivity bridge tests
7-
dfu: dfu tests
8-
fota: standard fota tests
9-
fullmfw_fota: fullmfw fota tests
10-
wifi: wifi location tests
11-
gnss: device used for GNSS tests
12-
traces: modem trace tests
3+
slow: marks tests as slow (deselect with '-m "not slow"')
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
##########################################################################################
2+
# Copyright (c) 2025 Nordic Semiconductor
3+
# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
4+
##########################################################################################
5+
6+
import pytest
7+
import time
8+
import os
9+
import functools
10+
from utils.flash_tools import flash_device, reset_device
11+
from utils.nrfcloud_fota import FWType, NRFCloudFOTAError
12+
import sys
13+
sys.path.append(os.getcwd())
14+
from utils.logger import get_logger
15+
16+
logger = get_logger()
17+
18+
MFW_202_FILEPATH = "artifacts/mfw_nrf91x1_2.0.2.zip"
19+
20+
# Stable version used for testing
21+
TEST_APP_VERSION = "foo"
22+
TEST_APP_BIN = "artifacts/stable_version_jan_2025-update-signed.bin"
23+
24+
DELTA_MFW_BUNDLEID = "MODEM*3471f88e*mfw_nrf91x1_2.0.2-FOTA-TEST"
25+
FULL_MFW_BUNDLEID = "MDM_FULL*124c2b20*mfw_nrf91x1_full_2.0.2"
26+
NEW_MFW_DELTA_VERSION = "mfw_nrf91x1_2.0.2-FOTA-TEST"
27+
MFW_202_VERSION = "mfw_nrf91x1_2.0.2"
28+
29+
DEVICE_MSG_TIMEOUT = 60 * 5
30+
APP_FOTA_TIMEOUT = 60 * 10
31+
FULL_MFW_FOTA_TIMEOUT = 60 * 30
32+
33+
34+
def await_nrfcloud(func, expected, field, timeout):
35+
start = time.time()
36+
logger.info(f"Awaiting {field} == {expected} in nrfcloud shadow...")
37+
while True:
38+
time.sleep(5)
39+
if time.time() - start > timeout:
40+
raise RuntimeError(f"Timeout awaiting {field} update")
41+
try:
42+
data = func()
43+
except Exception as e:
44+
logger.warning(f"Exception {e} during waiting for {field}")
45+
continue
46+
logger.debug(f"Reported {field}: {data}")
47+
if expected in data:
48+
break
49+
50+
def get_appversion(t91x_fota):
51+
shadow = t91x_fota.fota.get_device(t91x_fota.device_id)
52+
return shadow["state"]["reported"]["device"]["deviceInfo"]["appVersion"]
53+
54+
def get_modemversion(t91x_fota):
55+
shadow = t91x_fota.fota.get_device(t91x_fota.device_id)
56+
return shadow["state"]["reported"]["device"]["deviceInfo"]["modemFirmware"]
57+
58+
def run_fota_resumption(t91x_fota, fota_type):
59+
timeout_50_percent= APP_FOTA_TIMEOUT/2
60+
t91x_fota.uart.wait_for_str("50%", timeout=timeout_50_percent)
61+
logger.debug(f"Testing fota resumption on disconnect for {fota_type} fota")
62+
63+
patterns_lte_offline = ["network: Network connectivity lost"]
64+
patterns_lte_normal = ["network: Network connectivity established"]
65+
66+
# LTE disconnect
67+
t91x_fota.uart.flush()
68+
t91x_fota.uart.write("lte offline\r\n")
69+
t91x_fota.uart.wait_for_str(patterns_lte_offline, timeout=20)
70+
71+
# LTE reconnect
72+
t91x_fota.uart.flush()
73+
t91x_fota.uart.write("lte normal\r\n")
74+
t91x_fota.uart.wait_for_str(patterns_lte_normal, timeout=120)
75+
76+
t91x_fota.uart.wait_for_str("fota_download: Refuse fragment, restart with offset")
77+
t91x_fota.uart.wait_for_str("fota_download: Downloading from offset:")
78+
79+
@pytest.fixture
80+
def run_fota_fixture(t91x_fota, hex_file):
81+
def _run_fota(bundle_id="", fota_type="app", fotatimeout=APP_FOTA_TIMEOUT, new_version=TEST_APP_VERSION):
82+
flash_device(os.path.abspath(hex_file))
83+
t91x_fota.uart.xfactoryreset()
84+
t91x_fota.uart.flush()
85+
reset_device()
86+
t91x_fota.uart.wait_for_str("Connected to Cloud")
87+
88+
if fota_type == "app":
89+
bundle_id = t91x_fota.fota.upload_firmware(
90+
"nightly_test_app",
91+
TEST_APP_BIN,
92+
TEST_APP_VERSION,
93+
"Bundle used for nightly test",
94+
FWType.app,
95+
)
96+
logger.info(f"Uploaded file {TEST_APP_BIN}: bundleId: {bundle_id}")
97+
98+
try:
99+
t91x_fota.data['job_id'] = t91x_fota.fota.create_fota_job(t91x_fota.device_id, bundle_id)
100+
t91x_fota.data['bundle_id'] = bundle_id
101+
except NRFCloudFOTAError as e:
102+
pytest.skip(f"FOTA create_job REST API error: {e}")
103+
logger.info(f"Created FOTA Job (ID: {t91x_fota.data['job_id']})")
104+
105+
# Sleep a bit and trigger fota poll
106+
for i in range(3):
107+
try:
108+
time.sleep(30)
109+
t91x_fota.uart.write("zbus button_press\r\n")
110+
t91x_fota.uart.wait_for_str("nrf_cloud_fota_poll: Starting FOTA download")
111+
break
112+
except AssertionError:
113+
continue
114+
else:
115+
raise AssertionError(f"Fota update not available after {i} attempts")
116+
117+
118+
if fota_type == "app":
119+
run_fota_resumption(t91x_fota, "app")
120+
121+
await_nrfcloud(
122+
functools.partial(t91x_fota.fota.get_fota_status, t91x_fota.data['job_id']),
123+
"COMPLETED",
124+
"FOTA status",
125+
fotatimeout
126+
)
127+
try:
128+
if fota_type == "app":
129+
await_nrfcloud(
130+
functools.partial(get_appversion, t91x_fota),
131+
new_version,
132+
"appVersion",
133+
DEVICE_MSG_TIMEOUT
134+
)
135+
else:
136+
await_nrfcloud(
137+
functools.partial(get_modemversion, t91x_fota),
138+
new_version,
139+
"modemFirmware",
140+
DEVICE_MSG_TIMEOUT
141+
)
142+
except RuntimeError:
143+
logger.error(f"Version is not {new_version} after {DEVICE_MSG_TIMEOUT}s")
144+
145+
return _run_fota
146+
147+
148+
def test_app_fota(run_fota_fixture):
149+
'''
150+
Test application FOTA from nightly version to stable version
151+
'''
152+
run_fota_fixture() # Uses default parameters for app FOTA
153+
154+
def test_delta_mfw_fota(run_fota_fixture):
155+
'''
156+
Test delta modem FOTA on nrf9151
157+
'''
158+
try:
159+
run_fota_fixture(
160+
bundle_id=DELTA_MFW_BUNDLEID,
161+
fota_type="delta",
162+
new_version=NEW_MFW_DELTA_VERSION
163+
)
164+
finally:
165+
# Restore mfw202, no matter if test pass/fails
166+
flash_device(os.path.abspath(MFW_202_FILEPATH))
167+
168+
@pytest.mark.slow
169+
def test_full_mfw_fota(run_fota_fixture):
170+
'''
171+
Test full modem FOTA on nrf9151
172+
'''
173+
run_fota_fixture(
174+
bundle_id=FULL_MFW_BUNDLEID,
175+
fota_type="full",
176+
new_version=MFW_202_VERSION,
177+
fotatimeout=FULL_MFW_FOTA_TIMEOUT
178+
)

0 commit comments

Comments
 (0)