Skip to content

Commit c60f9de

Browse files
mpfz0rMarius Sturm
mpfz0r
authored and
Marius Sturm
committed
Add Etag caching for UpdateRegistration results (#322)
* Add Etag caching for UpdateRegistration results The response to the registration PUT request contains configuration assignments, which can be cached as well. Only run the assignment updates if either the backends or the assignments have changed. While here, replace `IsEmpty()` with an `NotModified` flag on each response struct. * Always record the updated configuration checksum Changes, like the configuration color won't render a new configuration. So the sidecar kept querying the API with the old checksum. Fixes #287
1 parent 591e771 commit c60f9de

File tree

3 files changed

+47
-42
lines changed

3 files changed

+47
-42
lines changed

‎api/graylog.go

+12-3
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ func RequestBackendList(httpClient *http.Client, checksum string, ctx *context.C
7070
backendResponse.Checksum = resp.Header.Get("Etag")
7171
switch {
7272
case resp.StatusCode == 304:
73-
log.Debug("[RequestBackendList] No backend update available.")
73+
backendResponse.NotModified = true
74+
log.Debug("[RequestBackendList] No update available.")
7475
case resp.StatusCode != 200:
7576
msg := "Bad response status from Graylog server"
7677
system.GlobalStatus.Set(backends.StatusError, msg)
@@ -121,7 +122,8 @@ func RequestConfiguration(
121122
log.Infof("[RequestConfiguration] %s", msg)
122123
return graylog.ResponseCollectorConfiguration{}, nil
123124
case resp.StatusCode == 304:
124-
log.Debug("[RequestConfiguration] No configuration update available, skipping update.")
125+
log.Debug("[RequestConfiguration] No update available, skipping update.")
126+
configurationResponse.NotModified = true
125127
case resp.StatusCode != 200:
126128
msg := "Bad response status from Graylog server"
127129
system.GlobalStatus.Set(backends.StatusError, msg+": "+err.Error())
@@ -134,7 +136,7 @@ func RequestConfiguration(
134136
return configurationResponse, nil
135137
}
136138

137-
func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *graylog.StatusRequest) (graylog.ResponseCollectorRegistration, error) {
139+
func UpdateRegistration(httpClient *http.Client, checksum string, ctx *context.Ctx, status *graylog.StatusRequest) (graylog.ResponseCollectorRegistration, error) {
138140
c := rest.NewClient(httpClient, ctx)
139141
c.BaseURL = ctx.ServerUrl
140142

@@ -171,6 +173,9 @@ func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *grayl
171173
}
172174

173175
r, err := c.NewRequest("PUT", "/sidecars/"+ctx.NodeId, nil, registration)
176+
if checksum != "" {
177+
r.Header.Add("If-None-Match", "\""+checksum+"\"")
178+
}
174179
if err != nil {
175180
log.Error("[UpdateRegistration] Can not initialize REST request")
176181
return graylog.ResponseCollectorRegistration{}, err
@@ -181,13 +186,17 @@ func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *grayl
181186
if resp != nil && resp.StatusCode == 400 && strings.Contains(err.Error(), "Unable to map property") {
182187
log.Error("[UpdateRegistration] Sending collector status failed. Disabling `send_status` as fallback! ", err)
183188
ctx.UserConfig.SendStatus = false
189+
} else if resp != nil && resp.StatusCode == 304 {
190+
log.Debug("[UpdateRegistration] No update available.")
191+
respBody.NotModified = true
184192
} else if resp != nil && resp.StatusCode != 202 {
185193
log.Error("[UpdateRegistration] Bad response from Graylog server: ", resp.Status)
186194
return graylog.ResponseCollectorRegistration{}, err
187195
} else if err != nil && err != io.EOF { // err is nil for GL 2.2 and EOF for 2.1 and earlier
188196
log.Error("[UpdateRegistration] Failed to report collector status to server: ", err)
189197
return graylog.ResponseCollectorRegistration{}, err
190198
}
199+
respBody.Checksum = resp.Header.Get("Etag")
191200

192201
// Update configuration if provided
193202
if respBody.Configuration != (graylog.ResponseCollectorRegistrationConfiguration{}) {

‎api/graylog/responses.go

+6-16
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ type ResponseCollectorRegistration struct {
2222
ConfigurationOverride bool `json:"configuration_override"`
2323
CollectorActions []ResponseCollectorAction `json:"actions,omitempty"`
2424
Assignments []assignments.ConfigurationAssignment `json:"assignments,omitempty"`
25+
Checksum string //Etag of the response
26+
NotModified bool
2527
}
2628

2729
type ResponseCollectorAction struct {
@@ -35,15 +37,9 @@ type ResponseCollectorRegistrationConfiguration struct {
3537
}
3638

3739
type ResponseBackendList struct {
38-
Backends []ResponseCollectorBackend `json:"collectors"`
39-
Checksum string //Etag of the response
40-
}
41-
42-
func (r *ResponseBackendList) IsEmpty() bool {
43-
if len(r.Backends) == 0 {
44-
return true
45-
}
46-
return false
40+
Backends []ResponseCollectorBackend `json:"collectors"`
41+
Checksum string //Etag of the response
42+
NotModified bool
4743
}
4844

4945
type ResponseCollectorBackend struct {
@@ -63,11 +59,5 @@ type ResponseCollectorConfiguration struct {
6359
Name string `json:"name"`
6460
Template string `json:"template"`
6561
Checksum string //Etag of the response
66-
}
67-
68-
func (r *ResponseCollectorConfiguration) IsEmpty() bool {
69-
if len(r.Template) == 0 {
70-
return true
71-
}
72-
return false
62+
NotModified bool
7363
}

‎services/periodicals.go

+29-23
Original file line numberDiff line numberDiff line change
@@ -40,53 +40,59 @@ func StartPeriodicals(context *context.Ctx) {
4040
go func() {
4141
configChecksums := make(map[string]string)
4242
backendChecksum := ""
43+
assignmentChecksum := ""
4344
logOnce := true
4445
for {
4546
time.Sleep(time.Duration(context.UserConfig.UpdateInterval) * time.Second)
4647

4748
// registration response contains configuration assignments
48-
response, err := updateCollectorRegistration(httpClient, context)
49+
response, err := updateCollectorRegistration(httpClient, assignmentChecksum, context)
4950
if err != nil {
5051
continue
5152
}
53+
assignmentChecksum = response.Checksum
5254
// backend list is needed before configuration assignments are updated
53-
backendChecksum, err = fetchBackendList(httpClient, backendChecksum, context)
55+
backendResponse, err := fetchBackendList(httpClient, backendChecksum, context)
5456
if err != nil {
5557
continue
5658
}
57-
assignments.Store.Update(response.Assignments)
58-
// create process instances
59-
daemon.Daemon.SyncWithAssignments(configChecksums, context)
60-
// test for new or updated configurations and start the corresponding collector
61-
if assignments.Store.Len() == 0 {
62-
if logOnce {
63-
log.Info("No configurations assigned to this instance. Skipping configuration request.")
64-
logOnce = false
59+
backendChecksum = backendResponse.Checksum
60+
61+
if !response.NotModified || !backendResponse.NotModified {
62+
assignments.Store.Update(response.Assignments)
63+
// create process instances
64+
daemon.Daemon.SyncWithAssignments(configChecksums, context)
65+
// test for new or updated configurations and start the corresponding collector
66+
if assignments.Store.Len() == 0 {
67+
if logOnce {
68+
log.Info("No configurations assigned to this instance. Skipping configuration request.")
69+
logOnce = false
70+
}
71+
continue
72+
} else {
73+
logOnce = true
6574
}
66-
continue
67-
} else {
68-
logOnce = true
6975
}
7076
checkForUpdateAndRestart(httpClient, configChecksums, context)
7177
}
7278
}()
7379
}
7480

75-
// report collector status to Graylog server
76-
func updateCollectorRegistration(httpClient *http.Client, context *context.Ctx) (graylog.ResponseCollectorRegistration, error) {
81+
// report collector status to Graylog server and receive assignments
82+
func updateCollectorRegistration(httpClient *http.Client, checksum string, context *context.Ctx) (graylog.ResponseCollectorRegistration, error) {
7783
statusRequest := api.NewStatusRequest()
78-
return api.UpdateRegistration(httpClient, context, &statusRequest)
84+
return api.UpdateRegistration(httpClient, checksum, context, &statusRequest)
7985
}
8086

81-
func fetchBackendList(httpClient *http.Client, checksum string, ctx *context.Ctx) (string, error) {
87+
func fetchBackendList(httpClient *http.Client, checksum string, ctx *context.Ctx) (graylog.ResponseBackendList, error) {
8288
response, err := api.RequestBackendList(httpClient, checksum, ctx)
8389
if err != nil {
8490
log.Error("Can't fetch collector list from Graylog API: ", err)
85-
return "", err
91+
return response, err
8692
}
87-
if response.IsEmpty() {
93+
if response.NotModified {
8894
// etag match, skipping all other actions
89-
return response.Checksum, nil
95+
return response, nil
9096
}
9197

9298
backendList := []backends.Backend{}
@@ -95,7 +101,7 @@ func fetchBackendList(httpClient *http.Client, checksum string, ctx *context.Ctx
95101
}
96102
backends.Store.Update(backendList)
97103

98-
return response.Checksum, nil
104+
return response, nil
99105
}
100106

101107
// fetch configuration periodically
@@ -113,13 +119,13 @@ func checkForUpdateAndRestart(httpClient *http.Client, checksums map[string]stri
113119
return
114120
}
115121

116-
if response.IsEmpty() {
122+
if response.NotModified {
117123
// etag match, skip file render
118124
continue
119125
}
126+
checksums[backendId] = response.Checksum
120127

121128
if backend.RenderOnChange(backends.Backend{Template: response.Template}, context) {
122-
checksums[backendId] = response.Checksum
123129
if err, output := backend.ValidateConfigurationFile(context); err != nil {
124130
backend.SetStatusLogErrorf(err.Error())
125131
if output != "" {

0 commit comments

Comments
 (0)