Skip to content

Commit afb4729

Browse files
michel-latermanmichel-laterman
authored
Make benchmarks more consistent (#3295)
Call b.ResetTimer() directly before main benchmarks are run in order to ignore any overhead in setting up the benchmarks Restructure some of the benchmarks for more consistent performance. Use a nop logger is some benchmarks. Run benchstat after the individual benchmark jobs to get a summery of the benchmarks for the pr/main.
1 parent 16e1d29 commit afb4729

12 files changed

+109
-77
lines changed

‎.buildkite/pipeline.yml

+2
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ steps:
4646
command: ".buildkite/scripts/run_benchmark.sh pr"
4747
artifact_paths:
4848
- build/next.out
49+
- build/next.stat
4950
agents:
5051
provider: "gcp"
5152
machineType: "c2-standard-8"
@@ -55,6 +56,7 @@ steps:
5556
command: ".buildkite/scripts/run_benchmark.sh base"
5657
artifact_paths:
5758
- build/base.out
59+
- build/base.stat
5860
agents:
5961
provider: "gcp"
6062
machineType: "c2-standard-8"

‎.buildkite/scripts/run_benchmark.sh

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export BENCHMARK_ARGS="-count=8 -benchmem"
1515
if [[ ${TYPE} == "pr" ]]; then
1616
echo "Starting the go benchmark for the pull request"
1717
BENCH_BASE=next.out make benchmark
18+
BENCH_BASE=next.out make benchstat | tee build/next.stat
1819
BENCH=$(cat build/next.out)
1920
buildkite-agent annotate --style 'info' --context "gobench_pr" --append << _EOF_
2021
#### Benchmark for pull request
@@ -34,6 +35,7 @@ if [[ ${TYPE} == "base" ]]; then
3435
echo "Starting the go benchmark for the pull request"
3536
git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
3637
BENCH_BASE=base.out make benchmark
38+
BENCH_BASE=base.out make benchstat | tee build/base.stat
3739
BENCH=$(cat build/base.out)
3840
buildkite-agent annotate --style 'info' --context "gobench_base" --append << _EOF_
3941
#### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ BUILDER_IMAGE=docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-d
1818
#Benchmark related targets
1919
BENCH_BASE ?= benchmark-$(COMMIT).out
2020
BENCH_NEXT ?=
21-
BENCHMARK_ARGS := -count=8 -benchmem
21+
BENCHMARK_ARGS := -count=10 -benchtime=3s -benchmem
2222
BENCHMARK_PACKAGE ?= ./...
2323
BENCHMARK_FILTER ?= Bench
2424

‎internal/pkg/api/handleAck_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,19 @@ import (
2828
)
2929

3030
func BenchmarkMakeUpdatePolicyBody(b *testing.B) {
31-
b.ReportAllocs()
32-
3331
const policyID = "ed110be4-c2a0-42b8-adc0-94c2f0569207"
3432
const newRev = 2
3533
const coord = 1
3634

35+
b.ResetTimer()
36+
b.ReportAllocs()
37+
3738
for n := 0; n < b.N; n++ {
3839
makeUpdatePolicyBody(policyID, newRev, coord)
3940
}
4041
}
4142

4243
func TestMakeUpdatePolicyBody(t *testing.T) {
43-
4444
const policyID = "ed110be4-c2a0-42b8-adc0-94c2f0569207"
4545
const newRev = 2
4646
const coord = 1

‎internal/pkg/api/handleCheckin_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
3030

3131
"github.com/hashicorp/go-version"
32+
"github.com/rs/zerolog"
3233
"github.com/stretchr/testify/assert"
3334
"github.com/stretchr/testify/mock"
3435
"github.com/stretchr/testify/require"
@@ -631,7 +632,7 @@ func Benchmark_CheckinT_writeResponse(b *testing.B) {
631632
}
632633
ct := NewCheckinT(verCon, cfg, nil, nil, nil, nil, nil, nil, ftesting.NewMockBulk())
633634

634-
logger := testlog.SetLogger(b)
635+
logger := zerolog.Nop()
635636
req := &http.Request{
636637
Header: http.Header{
637638
"Accept-Encoding": []string{"gzip"},
@@ -657,7 +658,7 @@ func BenchmarkParallel_CheckinT_writeResponse(b *testing.B) {
657658
}
658659
ct := NewCheckinT(verCon, cfg, nil, nil, nil, nil, nil, nil, ftesting.NewMockBulk())
659660

660-
logger := testlog.SetLogger(b)
661+
logger := zerolog.Nop()
661662
req := &http.Request{
662663
Header: http.Header{
663664
"Accept-Encoding": []string{"gzip"},

‎internal/pkg/bulk/bulk_integration_test.go

+4-7
Original file line numberDiff line numberDiff line change
@@ -380,11 +380,9 @@ func benchmarkCreate(n int, b *testing.B) {
380380
}
381381

382382
func BenchmarkCreate(b *testing.B) {
383-
384383
benchmarks := []int{1, 64, 8192, 16384, 32768, 65536}
385384

386385
for _, n := range benchmarks {
387-
388386
bindFunc := func(n int) func(b *testing.B) {
389387
return func(b *testing.B) {
390388
benchmarkCreate(n, b)
@@ -398,8 +396,6 @@ func BenchmarkCreate(b *testing.B) {
398396
// Not a particularly useful benchmark, but gives some idea of memory overhead.
399397

400398
func benchmarkCRUD(n int, b *testing.B) {
401-
b.ReportAllocs()
402-
403399
ctx, cn := context.WithCancel(context.Background())
404400
defer cn()
405401
ctx = testlog.SetLogger(b).WithContext(ctx)
@@ -415,8 +411,11 @@ func benchmarkCRUD(n int, b *testing.B) {
415411
ch := make(chan error, n)
416412
var wait sync.WaitGroup
417413
wait.Add(n)
418-
for i := 0; i < n; i++ {
419414

415+
b.ResetTimer()
416+
b.ReportAllocs()
417+
418+
for i := 0; i < n; i++ {
420419
go func() {
421420
defer wait.Done()
422421

@@ -466,11 +465,9 @@ func benchmarkCRUD(n int, b *testing.B) {
466465
}
467466

468467
func BenchmarkCRUD(b *testing.B) {
469-
470468
benchmarks := []int{1, 64, 8192, 16384, 32768, 65536}
471469

472470
for _, n := range benchmarks {
473-
474471
bindFunc := func(n int) func(b *testing.B) {
475472
return func(b *testing.B) {
476473
benchmarkCRUD(n, b)

‎internal/pkg/bulk/bulk_test.go

+3-7
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
"testing"
1717

1818
"github.com/elastic/fleet-server/v7/internal/pkg/apikey"
19-
testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
2019
"github.com/rs/zerolog"
2120
)
2221

@@ -305,12 +304,10 @@ func TestCancelCtxChildBulker(t *testing.T) {
305304
}
306305

307306
func benchmarkMockBulk(b *testing.B, samples [][]byte) {
308-
b.ReportAllocs()
309307
mock := &mockBulkTransport{}
310308

311309
ctx, cancelF := context.WithCancel(context.Background())
312310
defer cancelF()
313-
ctx = testlog.SetLogger(b).WithContext(ctx)
314311

315312
n := len(samples)
316313
bulker := NewBulker(mock, nil, WithFlushThresholdCount(n))
@@ -334,8 +331,10 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) {
334331

335332
var wait sync.WaitGroup
336333
wait.Add(n)
337-
for i := 0; i < n; i++ {
338334

335+
b.ResetTimer()
336+
b.ReportAllocs()
337+
for i := 0; i < n; i++ {
339338
go func(sampleData []byte) {
340339
defer wait.Done()
341340

@@ -360,7 +359,6 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) {
360359
// Delete
361360
err = bulker.Delete(ctx, index, id)
362361
if err != nil {
363-
b.Logf("Delete failed index: %s id: %s", index, id)
364362
b.Error(err)
365363
}
366364
}
@@ -373,7 +371,6 @@ func benchmarkMockBulk(b *testing.B, samples [][]byte) {
373371
}
374372

375373
func BenchmarkMockBulk(b *testing.B) {
376-
377374
benchmarks := []int{1, 8, 64, 4096, 32768}
378375

379376
// Create the samples outside the loop to avoid accounting
@@ -391,7 +388,6 @@ func BenchmarkMockBulk(b *testing.B) {
391388
}
392389

393390
for _, n := range benchmarks {
394-
395391
bindFunc := func(n int) func(b *testing.B) {
396392
return func(b *testing.B) {
397393
benchmarkMockBulk(b, samples[:n])

‎internal/pkg/bulk/opMulti_integration_test.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,9 @@ import (
1515
testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
1616
)
1717

18-
// This runs a series of CRUD operations through elastic.
18+
// benchmarkMultiUpdate runs a series of CRUD operations through elastic.
1919
// Not a particularly useful benchmark, but gives some idea of memory overhead.
20-
2120
func benchmarkMultiUpdate(n int, b *testing.B) {
22-
b.ReportAllocs()
23-
2421
ctx, cn := context.WithCancel(context.Background())
2522
defer cn()
2623
ctx = testlog.SetLogger(b).WithContext(ctx)
@@ -42,6 +39,9 @@ func benchmarkMultiUpdate(n int, b *testing.B) {
4239
b.Fatal(err)
4340
}
4441

42+
b.ResetTimer()
43+
b.ReportAllocs()
44+
4545
for j := 0; j < b.N; j++ {
4646
fields := UpdateFields{
4747
"dateval": time.Now().Format(time.RFC3339),
@@ -64,8 +64,9 @@ func benchmarkMultiUpdate(n int, b *testing.B) {
6464
}
6565
}
6666

67-
func BenchmarkMultiUpdate(b *testing.B) {
68-
67+
// BenchmarkMultiUpdateIntegration runs a benchmark for CRUD operations on a live ES instance
68+
// The results may be inconsistent due to the ES requirement.
69+
func BenchmarkMultiUpdateIntegration(b *testing.B) {
6970
benchmarks := []int{1, 64, 8192, 37268, 131072}
7071

7172
for _, n := range benchmarks {

‎internal/pkg/bulk/opMulti_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,9 @@ func BenchmarkMultiUpdateMock(b *testing.B) {
5050

5151
for _, n := range benchmarks {
5252
b.Run(strconv.Itoa(n), func(b *testing.B) {
53-
b.ReportAllocs()
5453
ctx := testlog.SetLogger(b).WithContext(context.Background())
54+
b.ResetTimer()
55+
b.ReportAllocs()
5556
for i := 0; i < b.N; i++ {
5657
if _, err := bulker.MUpdate(ctx, ops[:n]); err != nil {
5758
b.Fatal(err)

‎internal/pkg/checkin/bulk_test.go

+44-22
Original file line numberDiff line numberDiff line change
@@ -205,13 +205,8 @@ func validateTimestamp(tb testing.TB, start time.Time, ts string) {
205205
}
206206
}
207207

208-
func benchmarkBulk(n int, flush bool, b *testing.B) {
209-
ctx := testlog.SetLogger(b).WithContext(context.Background())
210-
b.ReportAllocs()
211-
208+
func benchmarkBulk(n int, b *testing.B) {
212209
mockBulk := ftesting.NewMockBulk()
213-
mockBulk.On("MUpdate", mock.Anything, mock.Anything, []bulk.Opt(nil)).Return([]bulk.BulkIndexerResponseItem{}, nil)
214-
215210
bc := NewBulk(mockBulk)
216211

217212
ids := make([]string, 0, n)
@@ -220,34 +215,61 @@ func benchmarkBulk(n int, flush bool, b *testing.B) {
220215
ids = append(ids, id)
221216
}
222217

218+
b.ResetTimer()
219+
b.ReportAllocs()
223220
for i := 0; i < b.N; i++ {
224-
225221
for _, id := range ids {
226222
err := bc.CheckIn(id, "", "", nil, nil, nil, "")
227223
if err != nil {
228224
b.Fatal(err)
229225
}
230226
}
227+
}
228+
}
231229

232-
if flush {
233-
err := bc.flush(ctx)
230+
func benchmarkFlush(n int, b *testing.B) {
231+
ctx := context.Background()
232+
mockBulk := ftesting.NewMockBulk()
233+
mockBulk.On("MUpdate", mock.Anything, mock.Anything, []bulk.Opt(nil)).Return([]bulk.BulkIndexerResponseItem{}, nil)
234+
bc := NewBulk(mockBulk)
235+
236+
ids := make([]string, 0, n)
237+
for i := 0; i < n; i++ {
238+
id := xid.New().String()
239+
ids = append(ids, id)
240+
}
241+
242+
b.ResetTimer()
243+
b.ReportAllocs()
244+
245+
for i := 0; i < b.N; i++ {
246+
b.StopTimer()
247+
for _, id := range ids {
248+
err := bc.CheckIn(id, "", "", nil, nil, nil, "")
234249
if err != nil {
235250
b.Fatal(err)
236251
}
237252
}
253+
b.StartTimer()
254+
255+
err := bc.flush(ctx)
256+
if err != nil {
257+
b.Fatal(err)
258+
}
238259
}
260+
239261
}
240262

241-
func BenchmarkBulk_1(b *testing.B) { benchmarkBulk(1, false, b) }
242-
func BenchmarkBulk_64(b *testing.B) { benchmarkBulk(64, false, b) }
243-
func BenchmarkBulk_8192(b *testing.B) { benchmarkBulk(8192, false, b) }
244-
func BenchmarkBulk_37268(b *testing.B) { benchmarkBulk(37268, false, b) }
245-
func BenchmarkBulk_131072(b *testing.B) { benchmarkBulk(131072, false, b) }
246-
func BenchmarkBulk_262144(b *testing.B) { benchmarkBulk(262144, false, b) }
247-
248-
func BenchmarkBulkFlush_1(b *testing.B) { benchmarkBulk(1, true, b) }
249-
func BenchmarkBulkFlush_64(b *testing.B) { benchmarkBulk(64, true, b) }
250-
func BenchmarkBulkFlush_8192(b *testing.B) { benchmarkBulk(8192, true, b) }
251-
func BenchmarkBulkFlush_37268(b *testing.B) { benchmarkBulk(37268, true, b) }
252-
func BenchmarkBulkFlush_131072(b *testing.B) { benchmarkBulk(131072, true, b) }
253-
func BenchmarkBulkFlush_262144(b *testing.B) { benchmarkBulk(262144, true, b) }
263+
func BenchmarkBulk_1(b *testing.B) { benchmarkBulk(1, b) }
264+
func BenchmarkBulk_64(b *testing.B) { benchmarkBulk(64, b) }
265+
func BenchmarkBulk_8192(b *testing.B) { benchmarkBulk(8192, b) }
266+
func BenchmarkBulk_37268(b *testing.B) { benchmarkBulk(37268, b) }
267+
func BenchmarkBulk_131072(b *testing.B) { benchmarkBulk(131072, b) }
268+
func BenchmarkBulk_262144(b *testing.B) { benchmarkBulk(262144, b) }
269+
270+
func BenchmarkFlush_1(b *testing.B) { benchmarkFlush(1, b) }
271+
func BenchmarkFlush_64(b *testing.B) { benchmarkFlush(64, b) }
272+
func BenchmarkFlush_8192(b *testing.B) { benchmarkFlush(8192, b) }
273+
func BenchmarkFlush_37268(b *testing.B) { benchmarkFlush(37268, b) }
274+
func BenchmarkFlush_131072(b *testing.B) { benchmarkFlush(131072, b) }
275+
func BenchmarkFlush_262144(b *testing.B) { benchmarkFlush(262144, b) }

0 commit comments

Comments
 (0)