Skip to content

Commit d8470ab

Browse files
jlind23jlind23
authored
Add go benchmark as part of the buildkite pipeline (#3141)
* add initial benchmark for PR * Add steps for comparing and baseline * Parse go benchmark results with gobenchdata * Allow gobenchdata failure to not block --------- Co-authored-by: Alexandros Sapranidis <alexandros@elastic.co>
1 parent c232532 commit d8470ab

File tree

6 files changed

+145
-102
lines changed

6 files changed

+145
-102
lines changed

‎.buildkite/pipeline.yml

+36
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,42 @@ steps:
3030
image: "${GO_AGENT_IMAGE}"
3131
cpu: "8"
3232
memory: "4G"
33+
34+
- group: "Performance test"
35+
key: "performance-test"
36+
steps:
37+
- label: "Run go benchmark for PR branch"
38+
key: "go-benchmark-pr"
39+
command: ".buildkite/scripts/run_benchmark.sh pr"
40+
artifact_paths:
41+
- build/next.out
42+
agents:
43+
provider: "gcp"
44+
machineType: "c2-standard-8"
45+
46+
- label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
47+
key: "go-benchmark-base"
48+
command: ".buildkite/scripts/run_benchmark.sh base"
49+
artifact_paths:
50+
- build/base.out
51+
agents:
52+
provider: "gcp"
53+
machineType: "c2-standard-8"
54+
55+
- label: "Compare results"
56+
key: "go-benchmark-compare"
57+
command: ".buildkite/scripts/run_benchmark.sh compare"
58+
artifact_paths:
59+
- build/failed_summary.md
60+
- build/failed_report.json
61+
- build/full_report.json
62+
depends_on:
63+
- go-benchmark-pr
64+
- go-benchmark-base
65+
agents:
66+
provider: "gcp"
67+
68+
depends_on: "check"
3369

3470
- group: "Run tests"
3571
key: "tests"

‎.buildkite/scripts/run_benchmark.sh

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/common.sh
6+
7+
add_bin_path
8+
9+
with_go
10+
11+
export TYPE=${1}
12+
#export BRANCH="${BUILDKITE_BRANCH}"
13+
export BENCHMARK_ARGS="-count=8 -benchmem"
14+
15+
if [[ ${TYPE} == "pr" ]]; then
16+
echo "Starting the go benchmark for the pull request"
17+
BENCH_BASE=next.out make benchmark
18+
BENCH=$(cat build/next.out)
19+
buildkite-agent annotate --style 'info' --context "gobench_pr" --append << _EOF_
20+
#### Benchmark for pull request
21+
<details><summary>go bench output</summary>
22+
23+
\`\`\`bash
24+
${BENCH}
25+
\`\`\`
26+
27+
</details>
28+
29+
Download <a href="artifact://build/next.out">next.out</a>
30+
_EOF_
31+
fi
32+
33+
if [[ ${TYPE} == "base" ]]; then
34+
echo "Starting the go benchmark for the pull request"
35+
git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
36+
BENCH_BASE=base.out make benchmark
37+
BENCH=$(cat build/base.out)
38+
buildkite-agent annotate --style 'info' --context "gobench_base" --append << _EOF_
39+
#### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
40+
<details><summary>go bench output for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}</summary>
41+
42+
\`\`\`bash
43+
${BENCH}
44+
\`\`\`
45+
46+
</details>
47+
48+
Download <a href="artifact://build/base.out">${BUILDKITE_PULL_REQUEST_BASE_BRANCH}.out</a>
49+
_EOF_
50+
fi
51+
52+
if [[ ${TYPE} == "compare" ]]; then
53+
echo "Comparing go benchmarks"
54+
go install go.bobheadxi.dev/gobenchdata@latest
55+
buildkite-agent artifact download "build/base.out" .
56+
buildkite-agent artifact download "build/next.out" .
57+
58+
cat build/base.out| gobenchdata --json build/base.json
59+
cat build/next.out| gobenchdata --json build/next.json
60+
set +e # suppress error handling of gobenchdata
61+
gobenchdata checks eval build/base.json build/next.json --json build/full_report.json
62+
status=$(jq -r '.Status' build/full_report.json)
63+
if [[ $status == "fail" ]]; then
64+
cat build/full_report.json| \
65+
jq 'del(.Checks.timePerOp.Diffs[]| select(.Status == "pass") )'| \
66+
tee build/failed_report.json
67+
gobenchdata checks report build/failed_report.json | tee build/failed_summary.md
68+
BENCH_COMPARE=$(cat build/failed_summary.md)
69+
buildkite-agent annotate --style 'error' --context "benchstat" --append << _EOF_
70+
#### Benchmark comparison
71+
<details><summary>Comparison table of benchmark results of HEAD compared to ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}</summary>
72+
73+
${BENCH_COMPARE}
74+
75+
</details>
76+
77+
Download <a href="artifact://build/failed_summary.md">failed_summary.md</a> , <a href="artifact://build/full_report.json">full_report.json</a>
78+
_EOF_
79+
#exit 1 # fail the build if the status is fail
80+
else
81+
BENCH_COMPARE=$(gobenchdata checks report build/full_report.json)
82+
buildkite-agent annotate --style 'success' --context "benchstat" --append << _EOF_
83+
#### Benchmark comparison
84+
<details><summary>No significant performance issue detect against ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}</summary>
85+
86+
${BENCH_COMPARE}
87+
88+
</details>
89+
90+
Download <a href="artifact://build/full_report.json">full_report.json</a>
91+
_EOF_
92+
fi
93+
fi
94+
95+

‎.github/workflows/benchmark.yml

-101
This file was deleted.

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,7 @@ dev-tools/cloud/terraform/*.tfvars*
2525

2626
.service_token*
2727
.kibana_service_token
28+
29+
30+
# direnv
31+
.envrc*

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ BUILDER_IMAGE=docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-d
1818
#Benchmark related targets
1919
BENCH_BASE ?= benchmark-$(COMMIT).out
2020
BENCH_NEXT ?=
21-
BENCHMARK_ARGS := -count=8
21+
BENCHMARK_ARGS := -count=8 -benchmem
2222
BENCHMARK_PACKAGE ?= ./...
2323
BENCHMARK_FILTER ?= Bench
2424

‎gobenchdata-checks.yml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
checks:
2+
- name: timePerOp
3+
description: |-
4+
This check is set to fail when there are benchmark tests that are slower than the defined threshold
5+
package: .
6+
benchmarks: []
7+
diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100
8+
thresholds:
9+
max: 10

0 commit comments

Comments
 (0)