Skip to content

Commit fef1c8a

Browse files
Ryo-not-rioRyo-not-rio
committed
ci: Add initial regression test workflow
1 parent ff65080 commit fef1c8a

File tree

6 files changed

+244
-0
lines changed

6 files changed

+244
-0
lines changed

‎.github/workflows/ci-aarch64.yml

+78
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ jobs:
9898
with:
9999
version: "17"
100100

101+
- name: setup python
102+
uses: actions/setup-python@v4
103+
with:
104+
python-version: '3.10'
105+
101106
- name: Clone ACL
102107
run: ${{ github.workspace }}/oneDNN/.github/automation/build_acl.sh
103108
env:
@@ -174,6 +179,79 @@ jobs:
174179
CTEST_PARALLEL_LEVEL: 6
175180
DYLD_LIBRARY_PATH: ${{ github.workspace }}/ComputeLibrary/build
176181
ONEDNN_THREADING: ${{ matrix.config.threading }}
182+
183+
## Regression test steps ##
184+
- name: Checkout oneDNN main
185+
if: ${{ matrix.config.build == "Release" }}
186+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
187+
with:
188+
ref: main
189+
path: oneDNN_main
190+
191+
# TODO :: Create separate pipeline to cache oneDNN main
192+
- name: Configure oneDNN main
193+
if: ${{ matrix.config.build == "Release" }}
194+
run: ${{ github.workspace }}/oneDNN/.github/automation/build_aarch64.sh
195+
working-directory: ${{ github.workspace }}/oneDNN_main
196+
env:
197+
ACL_ROOT_DIR: ${{ github.workspace }}/ComputeLibrary
198+
BUILD_TOOLSET: ${{ matrix.config.toolset }}
199+
CMAKE_BUILD_TYPE: ${{ matrix.config.build }}
200+
CMAKE_GENERATOR: Ninja
201+
GCC_VERSION: 13
202+
ONEDNN_ACTION: configure
203+
ONEDNN_TEST_SET: ${{ matrix.config.testset }}
204+
ONEDNN_THREADING: ${{ matrix.config.threading }}
205+
206+
- name: Build oneDNN main
207+
if: ${{ matrix.config.build == "Release" }}
208+
run: ${{ github.workspace }}/oneDNN/.github/automation/build_aarch64.sh
209+
working-directory: ${{ github.workspace }}/oneDNN_main
210+
env:
211+
ONEDNN_ACTION: build
212+
213+
- shell: bash
214+
if: ${{ matrix.config.build == "Release" }}
215+
run: |
216+
bash ${{ github.workspace }}/oneDNN/tests/regression/consistency_check.sh ${{ github.workspace }}/oneDNN_main/build/tests/benchdnn/benchdnn > consistency_1.txt
217+
bash ${{ github.workspace }}/oneDNN/tests/regression/consistency_check.sh ${{ github.workspace }}/oneDNN_main/build/tests/benchdnn/benchdnn > consistency_2.txt
218+
env:
219+
OMP_NUM_THREADS: 4
220+
DYLD_LIBRARY_PATH: ${{ github.workspace }}/ComputeLibrary/build
221+
222+
- name: Compare consistency check results
223+
if: ${{ matrix.config.build == "Release" }}
224+
id: consistency-check
225+
continue-on-error: true
226+
run: python ${{ github.workspace }}/oneDNN/tests/regression/benchdnn_comparison.py consistency_1.txt consistency_2.txt
227+
228+
- shell: bash
229+
if: ${{ matrix.config.build == "Release" }}
230+
run: |
231+
OMP_NUM_THREADS=4 bash ${{ github.workspace }}/oneDNN/tests/regression/bench_regression.sh ${{ github.workspace }}/oneDNN_main/build/tests/benchdnn/benchdnn >> main.txt
232+
OMP_NUM_THREADS=4 bash ${{ github.workspace }}/oneDNN/tests/regression/bench_regression.sh ${{ github.workspace }}/oneDNN/build/tests/benchdnn/benchdnn >> new.txt
233+
OMP_NUM_THREADS=16 bash ${{ github.workspace }}/oneDNN/tests/regression/bench_regression.sh ${{ github.workspace }}/oneDNN_main/build/tests/benchdnn/benchdnn >> main.txt
234+
OMP_NUM_THREADS=16 bash ${{ github.workspace }}/oneDNN/tests/regression/bench_regression.sh ${{ github.workspace }}/oneDNN/build/tests/benchdnn/benchdnn >> new.txt
235+
env:
236+
DYLD_LIBRARY_PATH: ${{ github.workspace }}/ComputeLibrary/build
237+
238+
- name: Compare regression test results
239+
if: ${{ matrix.config.build == "Release" }}
240+
id: regression-test
241+
continue-on-error: true
242+
run: python ${{ github.workspace }}/oneDNN/tests/regression/benchdnn_comparison.py main.txt new.txt
243+
244+
- name: Check consistency-check failure
245+
if: ${{ matrix.config.build == "Release" && steps.consistency-check.outputs.pass != 'True' && steps.regression-test.outputs.pass != 'True' }}
246+
run: |
247+
echo "::warnings title=consistency-check-failure::consistency check on main failed, ignoring regression test results!"
248+
249+
- name: Check regression test failure
250+
if: ${{ matrix.config.build == "Release" && steps.consistency-check.outputs.pass == 'True' && steps.regression-test.outputs.pass != 'True' }}
251+
run: |
252+
echo "::error title=regression-test-failure::some regression tests failed. Check the compare regression test results step for more details!"
253+
exit 1
254+
177255
# This job adds a check named "CI AArch64" that represents overall
178256
# workflow status and can be used in branch rulesets
179257
status:

‎tests/regression/bench_regression.sh

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#! /bin/bash
2+
3+
# *******************************************************************************
4+
# Copyright 2025 Arm Limited and affiliates.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
# *******************************************************************************
19+
20+
# Usage: bash bench_regression.sh {benchdnn_executable}
21+
22+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
23+
$1 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul
24+
$1 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv

‎tests/regression/benchdnn_comparison.py

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/python3
2+
3+
# *******************************************************************************
4+
# Copyright 2025 Arm Limited and affiliates.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
# *******************************************************************************
19+
20+
import sys
21+
import os
22+
23+
24+
def compare_two_benchdnn(file1, file2, tolerance=0.05):
25+
"""
26+
Compare two benchdnn output files
27+
"""
28+
with open(file1) as f:
29+
r1 = f.readlines()
30+
31+
with open(file2) as f:
32+
r2 = f.readlines()
33+
34+
# Trim non-formatted lines and split the prolem from time
35+
r1 = [x.split(",") for x in r1 if x[0:8] == "--mode=P"]
36+
r2 = [x.split(",") for x in r2 if x[0:8] == "--mode=P"]
37+
38+
# Convert to dict and trim \n
39+
r1 = [(x[0], float(x[1][:-1])) for x in r1]
40+
r2 = [(x[0], float(x[1][:-1])) for x in r2]
41+
42+
if len(r1) != len(r2):
43+
raise Exception("The number of benchdnn runs do not match")
44+
45+
print("%prb%,%-time(old)%,%-time(new)%,%passed%")
46+
47+
passed = True
48+
failed_tests = []
49+
for idx, item in enumerate(r1):
50+
prb, time1 = item
51+
if prb != r2[idx][0]:
52+
raise Exception(f"{prb} exists in {file1} but not {file2}")
53+
54+
res_str = f"{prb}, {time1}, {r2[idx][1]}"
55+
print(res_str)
56+
57+
if time1 != 0: # Incompatible tests would return 0 so avoid division by 0
58+
test_pass = (r2[idx][1] - time1) / time1 < tolerance
59+
if not test_pass:
60+
failed_tests.append(res_str)
61+
passed = False
62+
63+
if "GITHUB_OUTPUT" in os.environ:
64+
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
65+
f.write(f"pass={passed}")
66+
67+
if passed:
68+
print("Regression tests passed")
69+
else:
70+
print("\n----The following tests did not pass:----")
71+
print("\n".join(failed_tests) + "\n")
72+
raise Exception("Some regression tests did not pass")
73+
74+
if __name__ == "__main__":
75+
compare_two_benchdnn(sys.argv[1], sys.argv[2])

‎tests/regression/consistency_check.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#! /bin/bash
2+
3+
# *******************************************************************************
4+
# Copyright 2025 Arm Limited and affiliates.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
# *******************************************************************************
19+
20+
# Used for checking the fluctations in performance of a github actions runner
21+
# before performing the actual regression tests
22+
#
23+
# Usage: bash consistency_check.sh {benchdnn_executable}
24+
25+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
26+
$1 --matmul --mode=P --repeats-per-prb=20 --perf-template=%prb%,%-time% --dt=f32 128x300:300x128

‎tests/regression/inputs/conv

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# *******************************************************************************
2+
# Copyright 2025 Arm Limited and affiliates.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# *******************************************************************************
17+
18+
# From Resnet
19+
--reset
20+
--dir=FWD_D
21+
--dt=f32
22+
mb1_ic64oc256_ih200oh200kh1sh1dh0ph0_iw267ow267kw1sw1dw0pw0

‎tests/regression/inputs/matmul

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# *******************************************************************************
2+
# Copyright 2025 Arm Limited and affiliates.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# *******************************************************************************
17+
--reset
18+
--dt=bf16,f32
19+
384x1x384:384x384

0 commit comments

Comments
 (0)