Skip to content

Commit 7e800dd

Browse files
authored
Fix resource consumption issues (#125)
* Use varnishreload-compatible vcl names #81 (comment) * Honor varnishd max_vcl https://varnish-cache.org/docs/6.0/reference/varnishd.html#max-vcl
1 parent 51d20ba commit 7e800dd

File tree

3 files changed

+53
-10
lines changed

3 files changed

+53
-10
lines changed

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ require (
1313
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
1414
github.com/imdario/mergo v0.3.6 // indirect
1515
github.com/json-iterator/go v1.1.11 // indirect
16-
github.com/martin-helmich/go-varnish-client v0.2.1
16+
github.com/martin-helmich/go-varnish-client v0.2.2
1717
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
1818
github.com/modern-go/reflect2 v1.0.1 // indirect
1919
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect

go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ github.com/martin-helmich/go-varnish-client v0.2.0 h1:UNeB/8PQ+inkn1v57KLg2s3UkB
8080
github.com/martin-helmich/go-varnish-client v0.2.0/go.mod h1:BPZwupDItHsp9jLWUCFtchhJoJlt1RIjWoPEfeWTRdI=
8181
github.com/martin-helmich/go-varnish-client v0.2.1 h1:+NTZ+Io0dXMVrLJ534ABt8zr34sZop1fjt683qybmhg=
8282
github.com/martin-helmich/go-varnish-client v0.2.1/go.mod h1:mTUoKZn5fbP67YKPfKP2Zi8Y6A/+99B0oCQBFuHOhks=
83+
github.com/martin-helmich/go-varnish-client v0.2.2 h1:WBW/SwBPX+SEJN2nKh1/mhdgRqdNLgndxFE4RyD51hE=
84+
github.com/martin-helmich/go-varnish-client v0.2.2/go.mod h1:BUVEoli2BmESGMvlRiytvurkdZhJQwlzFtTBmG9mfFA=
8385
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
8486
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
8587
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -126,6 +128,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
126128
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
127129
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
128130
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
131+
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
129132
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
130133
golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490 h1:va0qYsIOza3Nlf2IncFyOql4/3XUq3vfge/Ad64bhlM=
131134
golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -194,6 +197,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
194197
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
195198
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
196199
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
200+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
197201
k8s.io/api v0.0.0-20181013054003-e94254e9898f h1:kLihNpdVw0H7RpDrS0XH9yI0+TI90p6YbaKlitJkaPc=
198202
k8s.io/api v0.0.0-20181013054003-e94254e9898f/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
199203
k8s.io/apimachinery v0.0.0-20181013010248-dcb88206cd7f h1:J4RMsuKRhuF+JxWL8Ip+w+lVxS/kypd+j0R93qPBD2c=

pkg/controller/watch.go

+48-9
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,18 @@ import (
55
"context"
66
"fmt"
77
"os/exec"
8+
"sort"
9+
"strconv"
10+
"strings"
811
"text/template"
12+
"time"
913

1014
"github.com/golang/glog"
1115
varnishclient "github.com/martin-helmich/go-varnish-client"
1216
)
1317

1418
func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd, errors chan<- error) {
15-
i := 0
16-
1719
for {
18-
i++
19-
2020
select {
2121
case tmplContents := <-v.vclTemplateUpdates:
2222
glog.Infof("VCL template has been updated")
@@ -29,7 +29,7 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd,
2929

3030
v.vclTemplate = tmpl
3131

32-
errors <- v.rebuildConfig(ctx, i)
32+
errors <- v.rebuildConfig(ctx)
3333

3434
case newConfig := <-v.frontendUpdates:
3535
glog.Infof("received new frontend configuration: %+v", newConfig)
@@ -40,14 +40,14 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd,
4040
v.varnishSignaller.SetEndpoints(v.frontend)
4141
}
4242

43-
errors <- v.rebuildConfig(ctx, i)
43+
errors <- v.rebuildConfig(ctx)
4444

4545
case newConfig := <-v.backendUpdates:
4646
glog.Infof("received new backend configuration: %+v", newConfig)
4747

4848
v.backend = newConfig
4949

50-
errors <- v.rebuildConfig(ctx, i)
50+
errors <- v.rebuildConfig(ctx)
5151

5252
case <-ctx.Done():
5353
errors <- ctx.Err()
@@ -56,7 +56,7 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd,
5656
}
5757
}
5858

59-
func (v *VarnishController) rebuildConfig(ctx context.Context, i int) error {
59+
func (v *VarnishController) rebuildConfig(ctx context.Context) error {
6060
buf := new(bytes.Buffer)
6161

6262
err := v.renderVCL(buf, v.frontend.Endpoints, v.frontend.Primary, v.backend.Endpoints, v.backend.Primary)
@@ -77,7 +77,46 @@ func (v *VarnishController) rebuildConfig(ctx context.Context, i int) error {
7777
return err
7878
}
7979

80-
configname := fmt.Sprintf("k8s-upstreamcfg-%d", i)
80+
maxVclParam, err := client.GetParameter(ctx, "max_vcl")
81+
if err != nil {
82+
return err
83+
}
84+
85+
maxVcl, err := strconv.Atoi(maxVclParam.Value)
86+
if err != nil {
87+
return err
88+
}
89+
90+
loadedVcl, err := client.ListVCL(ctx)
91+
if err != nil {
92+
return err
93+
}
94+
95+
availableVcl := make([]varnishclient.VCLConfig, 0)
96+
97+
for i := range loadedVcl {
98+
if loadedVcl[i].Status == varnishclient.VCLAvailable {
99+
availableVcl = append(availableVcl, loadedVcl[i])
100+
}
101+
}
102+
103+
if len(loadedVcl) >= maxVcl {
104+
// we're abusing the fact that "boot" < "reload"
105+
sort.Slice(availableVcl, func(i, j int) bool {
106+
return availableVcl[i].Name < availableVcl[j].Name
107+
})
108+
109+
for i := 0; i < len(loadedVcl)-maxVcl+1; i++ {
110+
glog.V(8).Infof("discarding VCL: %s", availableVcl[i].Name)
111+
112+
err = client.DiscardVCL(ctx, availableVcl[i].Name)
113+
if err != nil {
114+
return err
115+
}
116+
}
117+
}
118+
119+
configname := strings.ReplaceAll(time.Now().Format("reload_20060102_150405.00000"), ".", "_")
81120

82121
err = client.DefineInlineVCL(ctx, configname, vcl, varnishclient.VCLStateAuto)
83122
if err != nil {

0 commit comments

Comments
 (0)