From 2bddbdfcd3f742fe5245e4cc128d112a0eb5e64d Mon Sep 17 00:00:00 2001 From: Patric Vormstein Date: Thu, 9 Jan 2025 11:56:12 +0100 Subject: [PATCH 1/6] add global request size limit to OAS --- apidef/api_definitions.go | 1 + apidef/oas/middleware.go | 95 +++++++++++++++-- apidef/oas/middleware_test.go | 144 ++++++++++++++++++++++++++ gateway/mw_request_size_limit.go | 2 +- gateway/mw_request_size_limit_test.go | 47 +++++++++ 5 files changed, 281 insertions(+), 8 deletions(-) diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index c1a052ab4f9..08bfa4bcd7d 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -479,6 +479,7 @@ type VersionInfo struct { GlobalResponseHeadersDisabled bool `bson:"global_response_headers_disabled" json:"global_response_headers_disabled"` IgnoreEndpointCase bool `bson:"ignore_endpoint_case" json:"ignore_endpoint_case"` GlobalSizeLimit int64 `bson:"global_size_limit" json:"global_size_limit"` + GlobalSizeLimitDisabled bool `bson:"global_size_limit_disabled" json:"global_size_limit_disabled"` OverrideTarget string `bson:"override_target" json:"override_target"` } diff --git a/apidef/oas/middleware.go b/apidef/oas/middleware.go index eaa9e32cc6f..fddb589d39e 100644 --- a/apidef/oas/middleware.go +++ b/apidef/oas/middleware.go @@ -101,6 +101,9 @@ type Global struct { // TrafficLogs contains the configurations related to API level log analytics. TrafficLogs *TrafficLogs `bson:"trafficLogs,omitempty" json:"trafficLogs,omitempty"` + + // RequestSizeLimit contains the configuration related to limiting the global request size. + RequestSizeLimit *GlobalRequestSizeLimit `bson:"requestSizeLimit,omitempty" json:"requestSizeLimit,omitempty"` } // MarshalJSON is a custom JSON marshaler for the Global struct. It is implemented @@ -227,6 +230,8 @@ func (g *Global) Fill(api apidef.APIDefinition) { g.fillContextVariables(api) g.fillTrafficLogs(api) + + g.fillRequestSizeLimit(api) } func (g *Global) fillTrafficLogs(api apidef.APIDefinition) { @@ -240,6 +245,17 @@ func (g *Global) fillTrafficLogs(api apidef.APIDefinition) { } } +func (g *Global) fillRequestSizeLimit(api apidef.APIDefinition) { + if g.RequestSizeLimit == nil { + g.RequestSizeLimit = &GlobalRequestSizeLimit{} + } + + g.RequestSizeLimit.Fill(api) + if ShouldOmit(g.RequestSizeLimit) { + g.RequestSizeLimit = nil + } +} + func (g *Global) fillContextVariables(api apidef.APIDefinition) { if g.ContextVariables == nil { g.ContextVariables = &ContextVariables{} @@ -312,12 +328,7 @@ func (g *Global) ExtractTo(api *apidef.APIDefinition) { var resHeaderMeta apidef.HeaderInjectionMeta g.TransformResponseHeaders.ExtractTo(&resHeaderMeta) - if len(api.VersionData.Versions) == 0 { - api.VersionData.Versions = map[string]apidef.VersionInfo{ - Main: {}, - } - } - + requireMainVersion(api) vInfo := api.VersionData.Versions[Main] vInfo.GlobalHeadersDisabled = headerMeta.Disabled vInfo.GlobalHeaders = headerMeta.AddHeaders @@ -326,7 +337,9 @@ func (g *Global) ExtractTo(api *apidef.APIDefinition) { vInfo.GlobalResponseHeadersDisabled = resHeaderMeta.Disabled vInfo.GlobalResponseHeaders = resHeaderMeta.AddHeaders vInfo.GlobalResponseHeadersRemove = resHeaderMeta.DeleteHeaders - api.VersionData.Versions[Main] = vInfo + updateMainVersion(api, vInfo) + + g.extractRequestSizeLimitTo(api) } func (g *Global) extractTrafficLogsTo(api *apidef.APIDefinition) { @@ -340,6 +353,17 @@ func (g *Global) extractTrafficLogsTo(api *apidef.APIDefinition) { g.TrafficLogs.ExtractTo(api) } +func (g *Global) extractRequestSizeLimitTo(api *apidef.APIDefinition) { + if g.RequestSizeLimit == nil { + g.RequestSizeLimit = &GlobalRequestSizeLimit{} + defer func() { + g.RequestSizeLimit = nil + }() + } + + g.RequestSizeLimit.ExtractTo(api) +} + func (g *Global) extractContextVariablesTo(api *apidef.APIDefinition) { if g.ContextVariables == nil { g.ContextVariables = &ContextVariables{} @@ -1570,6 +1594,45 @@ func (t *TrafficLogs) ExtractTo(api *apidef.APIDefinition) { api.TagHeaders = t.TagHeaders } +// GlobalRequestSizeLimit holds configuration about the global limits for request sizes. +type GlobalRequestSizeLimit struct { + Enabled bool `bson:"enabled" json:"enabled"` + Value int64 `bson:"value" json:"value"` +} + +// Fill fills *GlobalRequestSizeLimit from apidef.APIDefinition. +func (g *GlobalRequestSizeLimit) Fill(api apidef.APIDefinition) { + ok := false + if api.VersionData.Versions != nil { + _, ok = api.VersionData.Versions[Main] + } + if !ok || api.VersionData.Versions[Main].GlobalSizeLimit == 0 { + g.Enabled = false + g.Value = 0 + return + } + + g.Enabled = !api.VersionData.Versions[Main].GlobalSizeLimitDisabled + g.Value = api.VersionData.Versions[Main].GlobalSizeLimit +} + +// ExtractTo extracts *GlobalRequestSizeLimit into *apidef.APIDefinition. +func (g *GlobalRequestSizeLimit) ExtractTo(api *apidef.APIDefinition) { + mainVersion := requireMainVersion(api) + defer func() { + updateMainVersion(api, mainVersion) + }() + + if g.Value == 0 { + mainVersion.GlobalSizeLimit = 0 + mainVersion.GlobalSizeLimitDisabled = true + return + } + + mainVersion.GlobalSizeLimitDisabled = !g.Enabled + mainVersion.GlobalSizeLimit = g.Value +} + // ContextVariables holds the configuration related to Tyk context variables. type ContextVariables struct { // Enabled enables context variables to be passed to Tyk middlewares. @@ -1586,3 +1649,21 @@ func (c *ContextVariables) Fill(api apidef.APIDefinition) { func (c *ContextVariables) ExtractTo(api *apidef.APIDefinition) { api.EnableContextVars = c.Enabled } + +func requireMainVersion(api *apidef.APIDefinition) apidef.VersionInfo { + if len(api.VersionData.Versions) == 0 { + api.VersionData.Versions = map[string]apidef.VersionInfo{ + Main: {}, + } + } + + if _, ok := api.VersionData.Versions[Main]; !ok { + api.VersionData.Versions[Main] = apidef.VersionInfo{} + } + + return api.VersionData.Versions[Main] +} + +func updateMainVersion(api *apidef.APIDefinition, updatedMainVersion apidef.VersionInfo) { + api.VersionData.Versions[Main] = updatedMainVersion +} diff --git a/apidef/oas/middleware_test.go b/apidef/oas/middleware_test.go index d7878a4ca23..322dd37d8d1 100644 --- a/apidef/oas/middleware_test.go +++ b/apidef/oas/middleware_test.go @@ -1086,3 +1086,147 @@ func TestContextVariables(t *testing.T) { } }) } + +func TestGlobalRequestSizeLimit(t *testing.T) { + t.Parallel() + t.Run("fill", func(t *testing.T) { + t.Parallel() + testcases := []struct { + title string + input apidef.APIDefinition + expected *GlobalRequestSizeLimit + }{ + { + title: "no versions", + input: apidef.APIDefinition{}, + expected: nil, + }, + { + title: "no main version", + input: apidef.APIDefinition{ + VersionData: apidef.VersionData{ + Versions: map[string]apidef.VersionInfo{ + "NotMain": {}, + }, + }, + }, + expected: nil, + }, + { + title: "request size limit set to 0 (disabled)", + input: apidef.APIDefinition{ + VersionData: apidef.VersionData{ + Versions: map[string]apidef.VersionInfo{ + Main: { + GlobalSizeLimit: 0, + GlobalSizeLimitDisabled: false, + }, + }, + }, + }, + expected: nil, + }, + { + title: "request size limit set to some value (enabled)", + input: apidef.APIDefinition{ + VersionData: apidef.VersionData{ + Versions: map[string]apidef.VersionInfo{ + Main: { + GlobalSizeLimit: 5000, + GlobalSizeLimitDisabled: false, + }, + }, + }, + }, + expected: &GlobalRequestSizeLimit{ + Enabled: true, + Value: 5000, + }, + }, + { + title: "request size limit set to some value but disabled", + input: apidef.APIDefinition{ + VersionData: apidef.VersionData{ + Versions: map[string]apidef.VersionInfo{ + Main: { + GlobalSizeLimit: 5000, + GlobalSizeLimitDisabled: true, + }, + }, + }, + }, + expected: &GlobalRequestSizeLimit{ + Enabled: false, + Value: 5000, + }, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + g := new(Global) + g.Fill(tc.input) + + assert.Equal(t, tc.expected, g.RequestSizeLimit) + }) + } + }) + + t.Run("extractTo", func(t *testing.T) { + t.Parallel() + + testcases := []struct { + title string + input *GlobalRequestSizeLimit + expectedGlobalRequestSizeLimit int64 + expectedGlobalRequestSizeLimitDisabled bool + }{ + { + title: "request size limit set to 0 (disabled)", + input: &GlobalRequestSizeLimit{ + Enabled: true, + Value: 0, + }, + expectedGlobalRequestSizeLimit: 0, + expectedGlobalRequestSizeLimitDisabled: true, + }, + { + title: "request size limit set to a value (enabled)", + input: &GlobalRequestSizeLimit{ + Enabled: true, + Value: 5000, + }, + expectedGlobalRequestSizeLimit: 5000, + expectedGlobalRequestSizeLimitDisabled: false, + }, + { + title: "request size limit set to a value and disabled", + input: &GlobalRequestSizeLimit{ + Enabled: false, + Value: 5000, + }, + expectedGlobalRequestSizeLimit: 5000, + expectedGlobalRequestSizeLimitDisabled: true, + }, + } + + for _, tc := range testcases { + tc := tc // Creating a new 'tc' scoped to the loop + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + g := new(Global) + g.RequestSizeLimit = tc.input + + var apiDef apidef.APIDefinition + g.ExtractTo(&apiDef) + + assert.Equal(t, tc.expectedGlobalRequestSizeLimit, apiDef.VersionData.Versions[Main].GlobalSizeLimit) + assert.Equal(t, tc.expectedGlobalRequestSizeLimitDisabled, apiDef.VersionData.Versions[Main].GlobalSizeLimitDisabled) + }) + } + }) +} diff --git a/gateway/mw_request_size_limit.go b/gateway/mw_request_size_limit.go index 62a46c1a8f1..b6e32abee10 100644 --- a/gateway/mw_request_size_limit.go +++ b/gateway/mw_request_size_limit.go @@ -39,7 +39,7 @@ func (t *RequestSizeLimitMiddleware) Name() string { func (t *RequestSizeLimitMiddleware) EnabledForSpec() bool { for _, version := range t.Spec.VersionData.Versions { if len(version.ExtendedPaths.SizeLimit) > 0 || - version.GlobalSizeLimit > 0 { + (!version.GlobalSizeLimitDisabled && version.GlobalSizeLimit > 0) { return true } } diff --git a/gateway/mw_request_size_limit_test.go b/gateway/mw_request_size_limit_test.go index 27d1a786b6f..044b65a7c8b 100644 --- a/gateway/mw_request_size_limit_test.go +++ b/gateway/mw_request_size_limit_test.go @@ -9,6 +9,7 @@ import ( "testing" logrus "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/TykTechnologies/tyk/apidef" @@ -92,4 +93,50 @@ func TestRequestSizeLimit(t *testing.T) { require.Errorf(t, err, "Content length is required for this request") } }) + + t.Run("check enabled for spec", func(t *testing.T) { + createMiddleware := func(versionInfoUpdater func(*apidef.VersionInfo)) *RequestSizeLimitMiddleware { + logger, _ := logrus.NewNullLogger() + spec := ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + UpdateAPIVersion(spec, "v1", versionInfoUpdater) + })[0] + baseMid := &BaseMiddleware{ + Spec: spec, + logger: logger.WithContext(context.Background()), + } + return &RequestSizeLimitMiddleware{baseMid} + } + + t.Run("request size limit set to 0 (disabled)", func(t *testing.T) { + mw := createMiddleware(func(v *apidef.VersionInfo) { + v.GlobalSizeLimit = 0 + v.GlobalSizeLimitDisabled = false + }) + assert.False(t, mw.EnabledForSpec()) + }) + + t.Run("request size limit set to value but disabled", func(t *testing.T) { + mw := createMiddleware(func(v *apidef.VersionInfo) { + v.GlobalSizeLimit = 5000 + v.GlobalSizeLimitDisabled = true + }) + assert.False(t, mw.EnabledForSpec()) + }) + + t.Run("request size limit set to 0 and disabled", func(t *testing.T) { + mw := createMiddleware(func(v *apidef.VersionInfo) { + v.GlobalSizeLimit = 0 + v.GlobalSizeLimitDisabled = true + }) + assert.False(t, mw.EnabledForSpec()) + }) + + t.Run("request size limit set to value and enabled", func(t *testing.T) { + mw := createMiddleware(func(v *apidef.VersionInfo) { + v.GlobalSizeLimit = 5000 + v.GlobalSizeLimitDisabled = false + }) + assert.True(t, mw.EnabledForSpec()) + }) + }) } From 8890bc4bb02d9f83bc174dbd01f0edab43a5fa29 Mon Sep 17 00:00:00 2001 From: Patric Vormstein Date: Thu, 9 Jan 2025 12:05:07 +0100 Subject: [PATCH 2/6] define GlobalRequestSizeLimit as supported --- apidef/oas/oas_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apidef/oas/oas_test.go b/apidef/oas/oas_test.go index dc839f76cec..e0a2259c23b 100644 --- a/apidef/oas/oas_test.go +++ b/apidef/oas/oas_test.go @@ -193,6 +193,7 @@ func TestOAS_ExtractTo_ResetAPIDefinition(t *testing.T) { vInfo.GlobalHeadersDisabled = false vInfo.GlobalResponseHeadersDisabled = false vInfo.UseExtendedPaths = false + vInfo.GlobalSizeLimitDisabled = false vInfo.ExtendedPaths.Clear() @@ -229,7 +230,6 @@ func TestOAS_ExtractTo_ResetAPIDefinition(t *testing.T) { "APIDefinition.VersionData.Versions[0].ExtendedPaths.PersistGraphQL[0].Operation", "APIDefinition.VersionData.Versions[0].ExtendedPaths.PersistGraphQL[0].Variables[0]", "APIDefinition.VersionData.Versions[0].IgnoreEndpointCase", - "APIDefinition.VersionData.Versions[0].GlobalSizeLimit", "APIDefinition.UptimeTests.CheckList[0].CheckURL", "APIDefinition.UptimeTests.CheckList[0].Protocol", "APIDefinition.UptimeTests.CheckList[0].Timeout", From 095e6fff3c039003bfb48e22010dd113cb5ffd49 Mon Sep 17 00:00:00 2001 From: Patric Vormstein Date: Thu, 9 Jan 2025 12:12:39 +0100 Subject: [PATCH 3/6] update x-tyk-api-gateway.json --- apidef/oas/schema/x-tyk-api-gateway.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apidef/oas/schema/x-tyk-api-gateway.json b/apidef/oas/schema/x-tyk-api-gateway.json index 76d4118829b..d450be55d83 100644 --- a/apidef/oas/schema/x-tyk-api-gateway.json +++ b/apidef/oas/schema/x-tyk-api-gateway.json @@ -552,6 +552,9 @@ }, "trafficLogs": { "$ref": "#/definitions/X-Tyk-TrafficLogs" + }, + "requestSizeLimit": { + "$ref": "#/definitions/X-Tyk-GlobalRequestSizeLimit" } } }, @@ -2000,6 +2003,21 @@ "enabled" ] }, + "X-Tyk-GlobalRequestSizeLimit": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "enabled", + "value" + ] + }, "X-Tyk-UInt": { "type": "integer", "minimum": 0 From 89134e6d447b611f4fe5f835ca2a198b61616b28 Mon Sep 17 00:00:00 2001 From: Patric Vormstein Date: Thu, 9 Jan 2025 12:35:15 +0100 Subject: [PATCH 4/6] update oas guide --- docs/dev/apidef-oas.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/dev/apidef-oas.md b/docs/dev/apidef-oas.md index 4b4d8664d4a..9d1bc76b29c 100644 --- a/docs/dev/apidef-oas.md +++ b/docs/dev/apidef-oas.md @@ -79,6 +79,12 @@ if ShouldOmit(u.RateLimit) { This ensures the field is reset to empty when not configured in the classic API definition. +### Working with `VersionData` +A `Main` version will be provided that can be used for `Fill`. +```go +api.VersionData.Versions[Main] +``` + ## Implement ExtractTo Method Pattern Similarly, follow this pattern with `ExtractTo`: @@ -94,6 +100,19 @@ if u.RateLimit == nil { u.RateLimit.ExtractTo(api) ``` +### Working with `VersionData` +There are 2 helper functions for `ExtractTo` that will help to handle `VersionData`. You can use them like this: +```go +func (g *GlobalRequestSizeLimit) ExtractTo(api *apidef.APIDefinition) { + mainVersion := requireMainVersion(api) + defer func() { + updateMainVersion(api, mainVersion) + }() + + // manipulate the Main VersionInfo here +} +``` + ## Write Tests Write tests for conversion functions. Refer to the example: From 1ccf6373e0e891e5362c22bf642e05880fb08140 Mon Sep 17 00:00:00 2001 From: Patric Vormstein Date: Thu, 9 Jan 2025 12:43:34 +0100 Subject: [PATCH 5/6] move helper code --- apidef/oas/middleware.go | 18 ------------------ apidef/oas/oasutil.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/apidef/oas/middleware.go b/apidef/oas/middleware.go index fddb589d39e..97d5341785b 100644 --- a/apidef/oas/middleware.go +++ b/apidef/oas/middleware.go @@ -1649,21 +1649,3 @@ func (c *ContextVariables) Fill(api apidef.APIDefinition) { func (c *ContextVariables) ExtractTo(api *apidef.APIDefinition) { api.EnableContextVars = c.Enabled } - -func requireMainVersion(api *apidef.APIDefinition) apidef.VersionInfo { - if len(api.VersionData.Versions) == 0 { - api.VersionData.Versions = map[string]apidef.VersionInfo{ - Main: {}, - } - } - - if _, ok := api.VersionData.Versions[Main]; !ok { - api.VersionData.Versions[Main] = apidef.VersionInfo{} - } - - return api.VersionData.Versions[Main] -} - -func updateMainVersion(api *apidef.APIDefinition, updatedMainVersion apidef.VersionInfo) { - api.VersionData.Versions[Main] = updatedMainVersion -} diff --git a/apidef/oas/oasutil.go b/apidef/oas/oasutil.go index e1b54bc6f87..640b664a9b1 100644 --- a/apidef/oas/oasutil.go +++ b/apidef/oas/oasutil.go @@ -3,6 +3,7 @@ package oas import ( "encoding/json" + "github.com/TykTechnologies/tyk/apidef" internalreflect "github.com/TykTechnologies/tyk/internal/reflect" ) @@ -27,3 +28,21 @@ func toStructIfMap(input interface{}, val interface{}) bool { // ShouldOmit is a compatibility alias. It may be removed in the future. var ShouldOmit = internalreflect.IsEmpty + +func requireMainVersion(api *apidef.APIDefinition) apidef.VersionInfo { + if len(api.VersionData.Versions) == 0 { + api.VersionData.Versions = map[string]apidef.VersionInfo{ + Main: {}, + } + } + + if _, ok := api.VersionData.Versions[Main]; !ok { + api.VersionData.Versions[Main] = apidef.VersionInfo{} + } + + return api.VersionData.Versions[Main] +} + +func updateMainVersion(api *apidef.APIDefinition, updatedMainVersion apidef.VersionInfo) { + api.VersionData.Versions[Main] = updatedMainVersion +} From 9f4752d2db40a277b21019f1838f04a5d8dc7cc5 Mon Sep 17 00:00:00 2001 From: Patric Vormstein Date: Thu, 9 Jan 2025 13:07:15 +0100 Subject: [PATCH 6/6] add comments --- apidef/oas/middleware.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apidef/oas/middleware.go b/apidef/oas/middleware.go index 97d5341785b..ae577a7c83f 100644 --- a/apidef/oas/middleware.go +++ b/apidef/oas/middleware.go @@ -1596,8 +1596,12 @@ func (t *TrafficLogs) ExtractTo(api *apidef.APIDefinition) { // GlobalRequestSizeLimit holds configuration about the global limits for request sizes. type GlobalRequestSizeLimit struct { - Enabled bool `bson:"enabled" json:"enabled"` - Value int64 `bson:"value" json:"value"` + // Enabled activates the Request Size Limit. + // Tyk classic API definition: `version_data.versions..global_size_limit_disabled`. + Enabled bool `bson:"enabled" json:"enabled"` + // Value contains the value of the request size limit. + // Tyk classic API definition: `version_data.versions..global_size_limit`. + Value int64 `bson:"value" json:"value"` } // Fill fills *GlobalRequestSizeLimit from apidef.APIDefinition.