Skip to content

Commit 579a439

Browse files
adityathebeadityathebe
committed
feat: metrics on temp cache
1 parent bd9160b commit 579a439

File tree

5 files changed

+37
-30
lines changed

5 files changed

+37
-30
lines changed

‎api/cache.go

+28
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package api
22

33
import (
44
"fmt"
5+
"strconv"
56
"strings"
67

78
"github.com/flanksource/commons/logger"
@@ -56,6 +57,11 @@ func (t *TempCache) Find(ctx ScrapeContext, lookup v1.ExternalID) (*models.Confi
5657
}
5758

5859
if _, ok := t.notFound[lookup.Key()]; ok {
60+
ctx.Counter("temp_cache_missing_hit",
61+
"scraper_id", ctx.ScraperID(),
62+
"incremental", strconv.FormatBool(ctx.IsIncrementalScrape()),
63+
"query_type", "external_id",
64+
).Add(1)
5965
return nil, nil
6066
}
6167

@@ -68,6 +74,12 @@ func (t *TempCache) Find(ctx ScrapeContext, lookup v1.ExternalID) (*models.Confi
6874
return t.Get(ctx, alias)
6975
}
7076

77+
ctx.Counter("temp_cache_miss",
78+
"scraper_id", ctx.ScraperID(),
79+
"incremental", strconv.FormatBool(ctx.IsIncrementalScrape()),
80+
"query_type", "external_id",
81+
).Add(1)
82+
7183
var result models.ConfigItem
7284
if err := lookup.Find(ctx.DB()).Find(&result).Error; err != nil {
7385
return nil, err
@@ -116,13 +128,29 @@ func (t *TempCache) Get(ctx ScrapeContext, id string, opts ...CacheOption) (*mod
116128
})
117129

118130
if _, notFound := t.notFound[id]; notFound && !optMap[IgnoreNotFound] {
131+
ctx.Counter("temp_cache_missing_hit",
132+
"scraper_id", ctx.ScraperID(),
133+
"incremental", strconv.FormatBool(ctx.IsIncrementalScrape()),
134+
"query_type", "id",
135+
).Add(1)
119136
return nil, nil
120137
}
121138

122139
if item, ok := t.items[id]; ok {
140+
ctx.Counter("temp_cache_hit",
141+
"scraper_id", ctx.ScraperID(),
142+
"incremental", strconv.FormatBool(ctx.IsIncrementalScrape()),
143+
"query_type", "id",
144+
).Add(1)
123145
return &item, nil
124146
}
125147

148+
ctx.Counter("temp_cache_miss",
149+
"scraper_id", ctx.ScraperID(),
150+
"incremental", strconv.FormatBool(ctx.IsIncrementalScrape()),
151+
"query_type", "id",
152+
).Add(1)
153+
126154
result := models.ConfigItem{}
127155

128156
if uuid.Validate(id) == nil {

‎db/config.go

-7
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,6 @@ func GetConfigItem(ctx api.ScrapeContext, extType, extID string) (*models.Config
4141
return &ci, nil
4242
}
4343

44-
// GetConfigItemFromID returns a single config item result
45-
func GetConfigItemFromID(ctx api.ScrapeContext, id string) (*models.ConfigItem, error) {
46-
var ci models.ConfigItem
47-
err := ctx.DB().Limit(1).Omit("config").Find(&ci, "id = ?", id).Error
48-
return &ci, dutydb.ErrorDetails(err)
49-
}
50-
5144
// CreateConfigItem inserts a new config item row in the db
5245
func CreateConfigItem(ctx api.ScrapeContext, ci *models.ConfigItem) error {
5346
if err := ctx.DB().Clauses(clause.OnConflict{UpdateAll: true}).Create(ci).Error; err != nil {

‎go.mod

-7
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,8 @@ require (
115115
github.com/asecurityteam/rolling v2.0.4+incompatible // indirect
116116
github.com/aws/aws-sdk-go-v2/config v1.27.29 // indirect
117117
github.com/aws/aws-sdk-go-v2/credentials v1.17.29 // indirect
118-
github.com/bahlo/generic-list-go v0.2.0 // indirect
119118
github.com/beorn7/perks v1.0.1 // indirect
120119
github.com/bmatcuk/doublestar/v4 v4.7.1 // indirect
121-
github.com/buger/jsonparser v1.1.1 // indirect
122120
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
123121
github.com/cert-manager/cert-manager v1.9.0 // indirect
124122
github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -160,7 +158,6 @@ require (
160158
github.com/hashicorp/hcl/v2 v2.21.0 // indirect
161159
github.com/henvic/httpretty v0.1.3 // indirect
162160
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
163-
github.com/invopop/jsonschema v0.12.0 // indirect
164161
github.com/itchyny/gojq v0.12.16 // indirect
165162
github.com/itchyny/timefmt-go v0.1.6 // indirect
166163
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect
@@ -204,11 +201,7 @@ require (
204201
github.com/vadimi/go-http-ntlm v1.0.3 // indirect
205202
github.com/vadimi/go-http-ntlm/v2 v2.4.1 // indirect
206203
github.com/vadimi/go-ntlm v1.2.1 // indirect
207-
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
208204
github.com/xanzy/ssh-agent v0.3.3 // indirect
209-
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
210-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
211-
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
212205
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
213206
github.com/yuin/gopher-lua v1.1.1 // indirect
214207
github.com/yusufpapurcu/wmi v1.2.4 // indirect

‎go.sum

-14
Original file line numberDiff line numberDiff line change
@@ -403,8 +403,6 @@ github.com/aws/aws-sdk-go-v2/service/support v1.24.3 h1:Bbesu6YZvEYACyydELMwUTYY
403403
github.com/aws/aws-sdk-go-v2/service/support v1.24.3/go.mod h1:NvXUhACskXZ2tiXzECpC/97xKzyY7/Wcc1ug5rla7kY=
404404
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
405405
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
406-
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
407-
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
408406
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
409407
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
410408
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -421,8 +419,6 @@ github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0
421419
github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
422420
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
423421
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
424-
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
425-
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
426422
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
427423
github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI=
428424
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
@@ -838,8 +834,6 @@ github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h
838834
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
839835
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
840836
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
841-
github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
842-
github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
843837
github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
844838
github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
845839
github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
@@ -1241,16 +1235,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
12411235
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
12421236
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
12431237
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
1244-
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
1245-
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
12461238
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
12471239
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
1248-
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
1249-
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
1250-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
1251-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
1252-
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
1253-
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
12541240
github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
12551241
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
12561242
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=

‎scrapers/run_test.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@ var _ = Describe("Scrapers test", Ordered, func() {
485485
Expect(err).To(BeNil())
486486
Expect(configItemID).ToNot(BeNil())
487487

488-
storedConfigItem, err = db.GetConfigItemFromID(ctx, configItemID.ID)
488+
storedConfigItem, err = GetConfigItemFromID(ctx, configItemID.ID)
489489
Expect(err).To(BeNil())
490490
Expect(storedConfigItem).ToNot(BeNil())
491491
})
@@ -517,7 +517,7 @@ var _ = Describe("Scrapers test", Ordered, func() {
517517
Expect(err).To(BeNil())
518518
Expect(configItemID).ToNot(BeNil())
519519

520-
configItem, err := db.GetConfigItemFromID(ctx, configItemID.ID)
520+
configItem, err := GetConfigItemFromID(ctx, configItemID.ID)
521521
Expect(err).To(BeNil())
522522
Expect(storedConfigItem).ToNot(BeNil())
523523

@@ -693,3 +693,10 @@ func createRandomConfigMap(name string) *apiv1.ConfigMap {
693693

694694
return cm1
695695
}
696+
697+
// GetConfigItemFromID returns a single config item result
698+
func GetConfigItemFromID(ctx api.ScrapeContext, id string) (*models.ConfigItem, error) {
699+
var ci models.ConfigItem
700+
err := ctx.DB().Limit(1).Omit("config").Find(&ci, "id = ?", id).Error
701+
return &ci, err
702+
}

0 commit comments

Comments
 (0)