Skip to content

Commit 6f1a913

Browse files
Ryo-not-rioRyo-not-rio
committed
ci: aarch64: Initial nightly performance test
Adding a comprehensive performance test to the nightly pipeline. For now, matmul, conv, eltwise and reorders are tested but more tests are to be added in the future. Other minor changes: - Fix benchdnn comparison script - Update precommit performance tests list - Add fast math tests
1 parent ca6c134 commit 6f1a913

17 files changed

+532
-54
lines changed

‎.github/automation/aarch64/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"dependencies": {
33
"acl": "v25.02",
44
"gcc": "13",
5-
"clang": "17"
5+
"clang": "17",
6+
"onednn-base": "v3.5.3"
67
}
7-
}
8+
}

‎.github/automation/performance/bench_fast_math_performance.sh

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#! /bin/bash
2+
3+
# *******************************************************************************
4+
# Copyright 2025 Arm Limited and affiliates.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
# *******************************************************************************
19+
20+
# Usage: bash bench_pr_performance.sh {baseline_benchdnn_executable} {benchdnn_executable} {baseline_results_file} {new_results_file}
21+
22+
IFS=$'\n' # Prevents shuffling from using spaces as delimiters
23+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
24+
25+
TESTS=(
26+
"ONEDNN_DEFAULT_FPMATH_MODE=BF16 $1 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul >> $3"
27+
"ONEDNN_DEFAULT_FPMATH_MODE=BF16 $2 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul >> $4"
28+
"ONEDNN_DEFAULT_FPMATH_MODE=BF16 $1 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv >> $3"
29+
"ONEDNN_DEFAULT_FPMATH_MODE=BF16 $2 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv >> $4"
30+
)
31+
32+
for i in {1..5}
33+
do
34+
echo "Testing loop ${i} / 5..."
35+
36+
TESTS=( $(shuf -e "${TESTS[@]}") )
37+
38+
for test in "${TESTS[@]}"
39+
do
40+
echo "Starting ${test}"
41+
SECONDS=0
42+
eval $test
43+
duration=$SECONDS
44+
echo "Completed in $((duration / 60)):$((duration % 60))"
45+
done
46+
done

‎.github/automation/performance/bench_nightly_performance.sh

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#! /bin/bash
2+
3+
# *******************************************************************************
4+
# Copyright 2025 Arm Limited and affiliates.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
# *******************************************************************************
19+
20+
# Usage: bash bench_nightly_performance.sh {baseline_benchdnn_executable} {benchdnn_executable} {baseline_results_file} {new_results_file}
21+
22+
IFS=$'\n' # Prevents shuffling from using spaces as delimiters
23+
24+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
25+
26+
TESTS=(
27+
"$1 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul_nightly >> $3"
28+
"$2 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul_nightly >> $4"
29+
"$1 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv_nightly >> $3"
30+
"$2 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv_nightly >> $4"
31+
"$1 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise_nightly >> $3"
32+
"$2 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise_nightly >> $4"
33+
"$1 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder_nightly >> $3"
34+
"$2 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder_nightly >> $4"
35+
)
36+
37+
for i in {1..5}
38+
do
39+
echo "Testing loop ${i} / 5..."
40+
41+
TESTS=( $(shuf -e "${TESTS[@]}") )
42+
43+
for test in "${TESTS[@]}"
44+
do
45+
echo "Starting ${test}"
46+
SECONDS=0
47+
eval $test
48+
duration=$SECONDS
49+
echo "Completed in $((duration / 60)):$((duration % 60))"
50+
done
51+
done

‎.github/automation/performance/bench_performance.sh

-32
This file was deleted.

‎.github/automation/performance/bench_pr_performance.sh

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#! /bin/bash
2+
3+
# *******************************************************************************
4+
# Copyright 2025 Arm Limited and affiliates.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
# *******************************************************************************
19+
20+
# Usage: bash bench_pr_performance.sh {baseline_benchdnn_executable} {benchdnn_executable} {baseline_results_file} {new_results_file}
21+
22+
IFS=$'\n' # Prevents shuffling from using spaces as delimiters
23+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
24+
25+
TESTS=(
26+
"$1 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul >> $3"
27+
"$2 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul >> $4"
28+
"$1 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv >> $3"
29+
"$2 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv >> $4"
30+
"$1 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise >> $3"
31+
"$2 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise >> $4"
32+
"$1 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder >> $3"
33+
"$2 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder >> $4"
34+
)
35+
36+
for i in {1..5}
37+
do
38+
echo "Testing loop ${i} / 5..."
39+
40+
TESTS=( $(shuf -e "${TESTS[@]}") )
41+
42+
for test in "${TESTS[@]}"
43+
do
44+
echo "Starting ${test}"
45+
SECONDS=0
46+
eval $test
47+
duration=$SECONDS
48+
echo "Completed in $((duration / 60)):$((duration % 60))"
49+
done
50+
done

‎.github/automation/performance/benchdnn_comparison.py

+20-11
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import os
2222
from collections import defaultdict
2323
from scipy.stats import ttest_ind
24+
import warnings
25+
import statistics
2426

2527

2628
def compare_two_benchdnn(file1, file2, tolerance=0.05):
@@ -38,9 +40,9 @@ def compare_two_benchdnn(file1, file2, tolerance=0.05):
3840
r2 = [x.split(",") for x in r2 if x[0:8] == "--mode=P"]
3941

4042
if (len(r1) == 0) or (len(r2) == 0):
41-
raise Exception("One or both of the test results have zero lines")
43+
warnings.warn("One or both of the test results have zero lines")
4244
if len(r1) != len(r2):
43-
raise Exception("The number of benchdnn runs do not match")
45+
warnings.warn("The number of benchdnn runs do not match")
4446

4547
r1_samples = defaultdict(list)
4648
r2_samples = defaultdict(list)
@@ -50,26 +52,33 @@ def compare_two_benchdnn(file1, file2, tolerance=0.05):
5052
for k, v in r2:
5153
r2_samples[k].append(float(v[:-1]))
5254

53-
passed = True
5455
failed_tests = []
56+
times = {}
5557
for prb, r1_times in r1_samples.items():
5658
if prb not in r2_samples:
57-
raise Exception(f"{prb} exists in {file1} but not {file2}")
59+
warnings.warn(f"{prb} exists in {file1} but not {file2}")
60+
continue
61+
5862
r2_times = r2_samples[prb]
5963

6064
res = ttest_ind(r2_times, r1_times, alternative='greater')
61-
62-
if res.pvalue < 0.05:
63-
failed_tests.append(prb)
65+
r1_med = statistics.median(r1_times)
66+
r2_med = statistics.median(r2_times)
67+
times[prb] = (r1_med, r2_med)
68+
times_str = f" {times[prb][0]} vs {times[prb][1]}"
69+
70+
passed = res.pvalue > 0.05 or \
71+
((r2_med - r1_med) / r1_med < 0.1 and \
72+
(min(r2_times) - min(r1_times)) / min(r1_times) < 0.1)
73+
if not passed:
74+
failed_tests.append(prb + times_str)
6475
passed = False
6576

66-
print(prb + (" passed" if passed else " failed"))
67-
6877
if "GITHUB_OUTPUT" in os.environ:
6978
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
70-
print(f"pass={passed}", file=f)
79+
print(f"pass={not failed_tests}", file=f)
7180

72-
if passed:
81+
if not failed_tests:
7382
print("Regression tests passed")
7483
else:
7584
message = "\n----The following regression tests failed:----\n" + \

‎.github/automation/performance/inputs/conv

+1-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@
1515
# limitations under the License.
1616
# *******************************************************************************
1717

18-
# From Resnet
1918
--reset
2019
--dir=FWD_D
21-
--dt=bf16,f32
20+
--dt=f32
2221
mb1_ic64oc256_ih200oh200kh1sh1dh0ph0_iw267ow267kw1sw1dw0pw0

‎.github/automation/performance/inputs/conv_nightly

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# *******************************************************************************
2+
# Copyright 2025 Arm Limited and affiliates.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# *******************************************************************************
17+
--reset
18+
--batch=conv
19+
20+
--reset
21+
--dt=f32
22+
--alg=auto
23+
--dir=FWD_D,FWD_B
24+
--batch=shapes_resnet_50

‎.github/automation/performance/inputs/eltwise

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@
1818
--inplace=true
1919
--alg=gelu_erf
2020
--dir=FWD_D
21-
--dt=bf16
21+
--dt=f32,bf16
2222
--tag=abc
23-
1x1x1536
23+
1536x384

‎.github/automation/performance/inputs/eltwise_nightly

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# *******************************************************************************
2+
# Copyright 2025 Arm Limited and affiliates.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# *******************************************************************************
17+
18+
--reset
19+
--batch=eltwise
20+
21+
--reset
22+
23+
--dt=f32
24+
--tag=abx,axb
25+
--dir=FWD_D
26+
--attr-post-ops=,
27+
28+
## algs which do not support alpha and beta + relu with alpha=0
29+
--alpha=0 --beta=0
30+
--alg=exp,exp_dst,gelu_erf,gelu_tanh,relu_dst,tanh,tanh_dst
31+
384x384
32+
33+
## algs which support negative alpha
34+
--alpha=-2 --beta=0
35+
--alg=elu,relu,swish
36+
384x384
37+
38+
## algs which support alpha and beta
39+
--alpha=-2 --beta=3
40+
--alg=linear
41+
384x384

‎.github/automation/performance/inputs/matmul

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@
1919
--wtag=any
2020
--attr-post-ops=sum
2121
--dtag=ab
22-
--dt=f32,bf16
22+
--dt=f32
2323
1500x1536:1536x384

‎.github/automation/performance/inputs/matmul_nightly

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# *******************************************************************************
2+
# Copyright 2025 Arm Limited and affiliates.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
# *******************************************************************************
17+
--reset
18+
--batch=matmul
19+
20+
# Plain cases
21+
--reset
22+
--dt=f32,s8:s8:f32
23+
--bia-dt=f32,undef
24+
--bia_mask=2
25+
--batch=shapes_2d_ci
26+
--bia_mask=4
27+
--batch=shapes_3d
28+
29+
#f16
30+
--dt=f16:f16:f16
31+
--bia-dt=undef
32+
--bia_mask=2
33+
--batch=shapes_2d_ci
34+
--bia_mask=4
35+
--batch=shapes_3d

0 commit comments

Comments
 (0)