Skip to content

Commit 91d6601

Browse files
jirevwejirevwe
authored
Add allow and block IP lists (#2169)
* feat: add ip allow and block lists to dispatcher; update dispatcher config * feat: added more validation * chore: fix lint errors * feat: put ip rules behind a feature flag and license; add flags for ip rules; update tests * chore: update allow-list rules for testcon tests * chore: update allow-list rules for testcon tests * feat: use default transport when the feature is disabled. * feat: update tests
1 parent 52acc9e commit 91d6601

29 files changed

+535
-86
lines changed

‎api/handlers/endpoint.go

+28-20
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"github.com/frain-dev/convoy/internal/pkg/fflag"
78
"github.com/frain-dev/convoy/pkg/circuit_breaker"
89
"github.com/frain-dev/convoy/pkg/msgpack"
910
"net/http"
@@ -211,29 +212,31 @@ func (h *Handler) GetEndpoints(w http.ResponseWriter, r *http.Request) {
211212
return
212213
}
213214

214-
// fetch keys from redis and mutate endpoints slice
215-
keys := make([]string, len(endpoints))
216-
for i := 0; i < len(endpoints); i++ {
217-
keys[i] = fmt.Sprintf("breaker:%s", endpoints[i].UID)
218-
}
215+
if h.A.FFlag.CanAccessFeature(fflag.CircuitBreaker) && h.A.Licenser.CircuitBreaking() && len(endpoints) > 0 {
216+
// fetch keys from redis and mutate endpoints slice
217+
keys := make([]string, len(endpoints))
218+
for i := 0; i < len(endpoints); i++ {
219+
keys[i] = fmt.Sprintf("breaker:%s", endpoints[i].UID)
220+
}
219221

220-
cbs, err := h.A.Redis.MGet(r.Context(), keys...).Result()
221-
if err != nil {
222-
_ = render.Render(w, r, util.NewServiceErrResponse(err))
223-
return
224-
}
222+
cbs, err := h.A.Redis.MGet(r.Context(), keys...).Result()
223+
if err != nil {
224+
_ = render.Render(w, r, util.NewServiceErrResponse(err))
225+
return
226+
}
225227

226-
for i := 0; i < len(cbs); i++ {
227-
if cbs[i] != nil {
228-
str, ok := cbs[i].(string)
229-
if ok {
230-
var c circuit_breaker.CircuitBreaker
231-
asBytes := []byte(str)
232-
innerErr := msgpack.DecodeMsgPack(asBytes, &c)
233-
if innerErr != nil {
234-
continue
228+
for i := 0; i < len(cbs); i++ {
229+
if cbs[i] != nil {
230+
str, ok := cbs[i].(string)
231+
if ok {
232+
var c circuit_breaker.CircuitBreaker
233+
asBytes := []byte(str)
234+
innerErr := msgpack.DecodeMsgPack(asBytes, &c)
235+
if innerErr != nil {
236+
continue
237+
}
238+
endpoints[i].FailureRate = c.FailureRate
235239
}
236-
endpoints[i].FailureRate = c.FailureRate
237240
}
238241
}
239242
}
@@ -505,6 +508,11 @@ func (h *Handler) PauseEndpoint(w http.ResponseWriter, r *http.Request) {
505508
// @Security ApiKeyAuth
506509
// @Router /v1/projects/{projectID}/endpoints/{endpointID}/activate [post]
507510
func (h *Handler) ActivateEndpoint(w http.ResponseWriter, r *http.Request) {
511+
if !h.A.Licenser.CircuitBreaking() || !h.A.FFlag.CanAccessFeature(fflag.CircuitBreaker) {
512+
_ = render.Render(w, r, util.NewErrorResponse("feature not enabled", http.StatusBadRequest))
513+
return
514+
}
515+
508516
project, err := h.retrieveProject(r)
509517
if err != nil {
510518
_ = render.Render(w, r, util.NewErrorResponse(err.Error(), http.StatusBadRequest))

‎api/server_suite_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"bytes"
88
"encoding/json"
99
"fmt"
10+
"github.com/frain-dev/convoy/internal/pkg/fflag"
1011
"io"
1112
"math/rand"
1213
"net/http"
@@ -137,6 +138,7 @@ func buildServer() *ApplicationHandler {
137138
Redis: rd.Client(),
138139
Logger: logger,
139140
Cache: noopCache,
141+
FFlag: fflag.NewFFlag([]string{string(fflag.Prometheus), string(fflag.FullTextSearch)}),
140142
Rate: r,
141143
Licenser: noopLicenser.NewLicenser(),
142144
})

‎cmd/agent/agent.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ func startServerComponent(_ context.Context, a *cli.App) error {
137137
lo.WithError(err).Fatal("failed to initialize realm chain")
138138
}
139139

140-
flag := fflag.NewFFlag(&cfg)
140+
flag := fflag.NewFFlag(cfg.EnableFeatureFlag)
141141

142142
lvl, err := log.ParseLevel(cfg.Logger.Level)
143143
if err != nil {

‎cmd/ff/feature_flags.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func AddFeatureFlagsCommand() *cobra.Command {
2121
log.WithError(err).Fatal("Error fetching the config.")
2222
}
2323

24-
f := fflag2.NewFFlag(&cfg)
24+
f := fflag2.NewFFlag(cfg.EnableFeatureFlag)
2525
return f.ListFeatures()
2626
},
2727
PersistentPostRun: func(cmd *cobra.Command, args []string) {},

‎cmd/hooks/hooks.go

+20-2
Original file line numberDiff line numberDiff line change
@@ -519,13 +519,31 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) {
519519
c.RetentionPolicy.IsRetentionPolicyEnabled = retentionPolicyEnabled
520520
}
521521

522-
// Feature flags
522+
// CONVOY_ENABLE_FEATURE_FLAG
523523
fflag, err := cmd.Flags().GetStringSlice("enable-feature-flag")
524524
if err != nil {
525525
return nil, err
526526
}
527527
c.EnableFeatureFlag = fflag
528528

529+
// CONVOY_DISPATCHER_BLOCK_LIST
530+
ipBlockList, err := cmd.Flags().GetStringSlice("ip-block-list")
531+
if err != nil {
532+
return nil, err
533+
}
534+
if len(ipBlockList) > 0 {
535+
c.Dispatcher.BlockList = ipBlockList
536+
}
537+
538+
// CONVOY_DISPATCHER_ALLOW_LIST
539+
ipAllowList, err := cmd.Flags().GetStringSlice("ip-allow-list")
540+
if err != nil {
541+
return nil, err
542+
}
543+
if len(ipAllowList) > 0 {
544+
c.Dispatcher.AllowList = ipAllowList
545+
}
546+
529547
// tracing
530548
tracingProvider, err := cmd.Flags().GetString("tracer-type")
531549
if err != nil {
@@ -585,7 +603,7 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) {
585603

586604
}
587605

588-
flag := fflag2.NewFFlag(c)
606+
flag := fflag2.NewFFlag(c.EnableFeatureFlag)
589607
c.Metrics = config.MetricsConfiguration{
590608
IsEnabled: false,
591609
}

‎cmd/main.go

+5
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ func main() {
4949
var dbDatabase string
5050

5151
var fflag []string
52+
var ipAllowList []string
53+
var ipBLockList []string
5254
var enableProfiling bool
5355

5456
var redisPort int
@@ -105,6 +107,9 @@ func main() {
105107

106108
// misc
107109
c.Flags().StringSliceVar(&fflag, "enable-feature-flag", []string{}, "List of feature flags to enable e.g. \"full-text-search,prometheus\"")
110+
c.Flags().StringSliceVar(&ipAllowList, "ip-allow-list", []string{}, "List of IPs CIDRs to allow e.g. \" 0.0.0.0/0,127.0.0.0/8\"")
111+
c.Flags().StringSliceVar(&ipBLockList, "ip-block-list", []string{}, "List of IPs CIDRs to block e.g. \" 0.0.0.0/0,127.0.0.0/8\"")
112+
108113
c.Flags().IntVar(&instanceIngestRate, "instance-ingest-rate", 0, "Instance ingest Rate")
109114
c.Flags().IntVar(&apiRateLimit, "api-rate-limit", 0, "API rate limit")
110115

‎cmd/server/server.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ func startConvoyServer(a *cli.App) error {
107107
a.Logger.WithError(err).Fatal("failed to initialize realm chain")
108108
}
109109

110-
flag := fflag.NewFFlag(&cfg)
110+
flag := fflag.NewFFlag(cfg.EnableFeatureFlag)
111111

112112
if cfg.Server.HTTP.Port <= 0 {
113113
return errors.New("please provide the HTTP port in the convoy.json file")

‎cmd/worker/worker.go

+10-2
Original file line numberDiff line numberDiff line change
@@ -249,18 +249,26 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte
249249

250250
go memorystore.DefaultStore.Sync(ctx, interval)
251251

252+
featureFlag := fflag.NewFFlag(cfg.EnableFeatureFlag)
252253
newTelemetry := telemetry.NewTelemetry(lo, configuration,
253254
telemetry.OptionTracker(counter),
254255
telemetry.OptionBackend(pb),
255256
telemetry.OptionBackend(mb))
256257

257-
dispatcher, err := net.NewDispatcher(cfg.Server.HTTP.HttpProxy, a.Licenser, false)
258+
dispatcher, err := net.NewDispatcher(
259+
a.Licenser,
260+
featureFlag,
261+
net.LoggerOption(lo),
262+
net.ProxyOption(cfg.Server.HTTP.HttpProxy),
263+
net.AllowListOption(cfg.Dispatcher.AllowList),
264+
net.BlockListOption(cfg.Dispatcher.BlockList),
265+
net.InsecureSkipVerifyOption(cfg.Dispatcher.InsecureSkipVerify),
266+
)
258267
if err != nil {
259268
lo.WithError(err).Fatal("Failed to create new net dispatcher")
260269
return err
261270
}
262271

263-
featureFlag := fflag.NewFFlag(&cfg)
264272
var circuitBreakerManager *cb.CircuitBreakerManager
265273

266274
if featureFlag.CanAccessFeature(fflag.CircuitBreaker) {

‎config/config.go

+12
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,11 @@ var DefaultConfiguration = Configuration{
107107
SampleTime: 5,
108108
},
109109
},
110+
Dispatcher: DispatcherConfiguration{
111+
InsecureSkipVerify: true,
112+
AllowList: []string{"0.0.0.0/0", "::/0"},
113+
BlockList: []string{"127.0.0.0/8", "::1/128"},
114+
},
110115
InstanceIngestRate: 25,
111116
ApiRateLimit: 25,
112117
WorkerExecutionMode: DefaultExecutionMode,
@@ -388,6 +393,13 @@ type Configuration struct {
388393
WorkerExecutionMode ExecutionMode `json:"worker_execution_mode" envconfig:"CONVOY_WORKER_EXECUTION_MODE"`
389394
MaxRetrySeconds uint64 `json:"max_retry_seconds,omitempty" envconfig:"CONVOY_MAX_RETRY_SECONDS"`
390395
LicenseKey string `json:"license_key" envconfig:"CONVOY_LICENSE_KEY"`
396+
Dispatcher DispatcherConfiguration `json:"dispatcher"`
397+
}
398+
399+
type DispatcherConfiguration struct {
400+
InsecureSkipVerify bool `json:"insecure_skip_verify" envconfig:"CONVOY_DISPATCHER_INSECURE_SKIP_VERIFY"`
401+
AllowList []string `json:"allow_list" envconfig:"CONVOY_DISPATCHER_ALLOW_LIST"`
402+
BlockList []string `json:"block_list" envconfig:"CONVOY_DISPATCHER_BLOCK_LIST"`
391403
}
392404

393405
type PyroscopeConfiguration struct {

‎config/config_test.go

+15
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,11 @@ func TestLoadConfig(t *testing.T) {
178178
SampleTime: 5,
179179
},
180180
},
181+
Dispatcher: DispatcherConfiguration{
182+
InsecureSkipVerify: true,
183+
AllowList: []string{"0.0.0.0/0", "::/0"},
184+
BlockList: []string{"127.0.0.0/8", "::1/128"},
185+
},
181186
WorkerExecutionMode: DefaultExecutionMode,
182187
InstanceIngestRate: 25,
183188
ApiRateLimit: 25,
@@ -265,6 +270,11 @@ func TestLoadConfig(t *testing.T) {
265270
SampleTime: 5,
266271
},
267272
},
273+
Dispatcher: DispatcherConfiguration{
274+
InsecureSkipVerify: true,
275+
AllowList: []string{"0.0.0.0/0", "::/0"},
276+
BlockList: []string{"127.0.0.0/8", "::1/128"},
277+
},
268278
InstanceIngestRate: 25,
269279
ApiRateLimit: 25,
270280
WorkerExecutionMode: DefaultExecutionMode,
@@ -351,6 +361,11 @@ func TestLoadConfig(t *testing.T) {
351361
SampleTime: 5,
352362
},
353363
},
364+
Dispatcher: DispatcherConfiguration{
365+
InsecureSkipVerify: true,
366+
AllowList: []string{"0.0.0.0/0", "::/0"},
367+
BlockList: []string{"127.0.0.0/8", "::1/128"},
368+
},
354369
InstanceIngestRate: 25,
355370
ApiRateLimit: 25,
356371
WorkerExecutionMode: DefaultExecutionMode,

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ require (
205205
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
206206
github.com/shoenig/go-m1cpu v0.1.6 // indirect
207207
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
208+
github.com/stealthrocket/netjail v0.1.2 // indirect
208209
github.com/theupdateframework/notary v0.7.0 // indirect
209210
github.com/tidwall/match v1.1.1 // indirect
210211
github.com/tidwall/pretty v1.2.1 // indirect

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -1841,6 +1841,8 @@ github.com/spf13/viper v0.0.0-20150530192845-be5ff3e4840c/go.mod h1:A8kyI5cUJhb8
18411841
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
18421842
github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
18431843
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
1844+
github.com/stealthrocket/netjail v0.1.2 h1:nOgFLer7XrkYcn8cJk5kI9aUFRkV7LC/8VjmJ2GjBQU=
1845+
github.com/stealthrocket/netjail v0.1.2/go.mod h1:LmslfwZTxTchb7koch3C/MNvEzF111G9HwZQrT23No4=
18441846
github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
18451847
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
18461848
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=

‎internal/pkg/fflag/fflag.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package fflag
33
import (
44
"errors"
55
"fmt"
6-
"github.com/frain-dev/convoy/config"
76
"os"
87
"sort"
98
"text/tabwriter"
@@ -18,9 +17,10 @@ type (
1817
)
1918

2019
const (
20+
IpRules FeatureFlagKey = "ip-rules"
2121
Prometheus FeatureFlagKey = "prometheus"
22-
FullTextSearch FeatureFlagKey = "full-text-search"
2322
CircuitBreaker FeatureFlagKey = "circuit-breaker"
23+
FullTextSearch FeatureFlagKey = "full-text-search"
2424
)
2525

2626
type (
@@ -33,6 +33,7 @@ const (
3333
)
3434

3535
var DefaultFeaturesState = map[FeatureFlagKey]FeatureFlagState{
36+
IpRules: disabled,
3637
Prometheus: disabled,
3738
FullTextSearch: disabled,
3839
CircuitBreaker: disabled,
@@ -42,13 +43,15 @@ type FFlag struct {
4243
Features map[FeatureFlagKey]FeatureFlagState
4344
}
4445

45-
func NewFFlag(c *config.Configuration) *FFlag {
46+
func NewFFlag(enableFeatureFlags []string) *FFlag {
4647
f := &FFlag{
4748
Features: clone(DefaultFeaturesState),
4849
}
4950

50-
for _, flag := range c.EnableFeatureFlag {
51+
for _, flag := range enableFeatureFlags {
5152
switch flag {
53+
case string(IpRules):
54+
f.Features[IpRules] = enabled
5255
case string(Prometheus):
5356
f.Features[Prometheus] = enabled
5457
case string(FullTextSearch):

0 commit comments

Comments
 (0)