Skip to content

feat: Add LogGroupClass support by upgrading AWS SDK to v1.48.4 #369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: mainline
Choose a base branch
from
Open

feat: Add LogGroupClass support by upgrading AWS SDK to v1.48.4 #369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ After this step, run `make windows-release`. Then use with Fluent Bit on Windows
* `log_stream_name`: The name of the CloudWatch Log Stream that you want log records sent to. This value allows a template in the form of `$(variable)`. See section [Templating Log Group and Stream Names](#templating-log-group-and-stream-names) for more.
* `default_log_group_name`: This required variable is the fallback in case any variables in `log_group_name` fails to parse. Defaults to `fluentbit-default`.
* `default_log_stream_name`: This required variable is the fallback in case any variables in `log_stream_name` fails to parse. Defaults to `/fluentbit-default`.
* `log_group_class`: The storage class to use for newly created log groups. Valid values are `STANDARD` or `INFREQUENT_ACCESS`. Defaults to `STANDARD`. This setting only applies when creating new log groups with `auto_create_group` set to true.
* `log_stream_prefix`: (deprecated) Prefix for the Log Stream name. Setting this to `prefix-` is the same as setting `log_stream_name = prefix-$(tag)`.
* `log_key`: By default, the whole log record will be sent to CloudWatch. If you specify a key name with this option, then only the value of that key will be sent to CloudWatch. For example, if you are using the Fluentd Docker log driver, you can specify `log_key log` and only the log message will be sent to CloudWatch.
* `log_format`: An optional parameter that can be used to tell CloudWatch the format of the data. A value of `json/emf` enables CloudWatch to extract custom metrics embedded in a JSON payload. See the [Embedded Metric Format](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Embedded_Metric_Format_Specification.html).
Expand Down
18 changes: 16 additions & 2 deletions cloudwatch/cloudwatch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ type OutputPlugin struct {
runningInECS bool
uuid string
extraUserAgent string
logGroupClass string
}

// OutputPluginConfig is the input information used by NewOutputPlugin to create a new OutputPlugin
Expand All @@ -169,6 +170,7 @@ type OutputPluginConfig struct {
PluginInstanceID int
LogFormat string
ExtraUserAgent string
LogGroupClass string
}

// Validate checks the configuration input for an OutputPlugin instances
Expand All @@ -188,6 +190,10 @@ func (config OutputPluginConfig) Validate() error {
return fmt.Errorf("either log_stream_name or log_stream_prefix can be configured. They cannot be provided together")
}

if config.LogGroupClass != "" && config.LogGroupClass != "STANDARD" && config.LogGroupClass != "INFREQUENT_ACCESS" {
return fmt.Errorf("log_group_class must be either empty or one of: STANDARD, INFREQUENT_ACCESS")
}

return nil
}

Expand Down Expand Up @@ -245,6 +251,7 @@ func NewOutputPlugin(config OutputPluginConfig) (*OutputPlugin, error) {
runningInECS: runningInECS,
uuid: ksuid.New().String(),
extraUserAgent: config.ExtraUserAgent,
logGroupClass: config.LogGroupClass,
}, nil
}

Expand Down Expand Up @@ -601,10 +608,17 @@ func (output *OutputPlugin) createLogGroup(e *Event) error {
return nil
}

_, err := output.client.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
input := &cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String(e.group),
Tags: output.logGroupTags,
})
}

// Only set LogGroupClass if it's specified in the config
if output.logGroupClass != "" {
input.LogGroupClass = aws.String(output.logGroupClass)
}

_, err := output.client.CreateLogGroup(input)
if err == nil {
logrus.Infof("[cloudwatch %d] Created log group %s\n", output.PluginInstanceID, e.group)
return output.setLogGroupRetention(e.group)
Expand Down
73 changes: 73 additions & 0 deletions cloudwatch/cloudwatch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,27 @@ type configTest struct {

var (
configValidationTestCases = []configTest{
{
name: "ValidConfigurationWithLogGroupClass",
config: OutputPluginConfig{
Region: testRegion,
LogGroupName: testLogGroup,
LogStreamPrefix: testLogStreamPrefix,
LogGroupClass: "INFREQUENT_ACCESS",
},
isValidConfig: true,
},
{
name: "InvalidLogGroupClass",
config: OutputPluginConfig{
Region: testRegion,
LogGroupName: testLogGroup,
LogStreamPrefix: testLogStreamPrefix,
LogGroupClass: "INVALID_CLASS",
},
isValidConfig: false,
expectedError: "log_group_class must be either empty or one of: STANDARD, INFREQUENT_ACCESS",
},
{
name: "ValidConfiguration",
config: OutputPluginConfig{
Expand Down Expand Up @@ -1029,6 +1050,58 @@ func setupTimeout() *plugins.Timeout {
return timer
}

func TestCreateLogGroupWithLogGroupClass(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockCloudWatch := mock_cloudwatch.NewMockLogsClient(ctrl)

output := OutputPlugin{
client: mockCloudWatch,
logGroupClass: "INFREQUENT_ACCESS",
autoCreateGroup: true,
}

input := &cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String(testLogGroup),
LogGroupClass: aws.String("INFREQUENT_ACCESS"),
}

mockCloudWatch.EXPECT().CreateLogGroup(input).Return(&cloudwatchlogs.CreateLogGroupOutput{}, nil)

e := &Event{
group: testLogGroup,
}

err := output.createLogGroup(e)
assert.NoError(t, err)
}

func TestCreateLogGroupWithoutLogGroupClass(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockCloudWatch := mock_cloudwatch.NewMockLogsClient(ctrl)

output := OutputPlugin{
client: mockCloudWatch,
autoCreateGroup: true,
}

input := &cloudwatchlogs.CreateLogGroupInput{
LogGroupName: aws.String(testLogGroup),
}

mockCloudWatch.EXPECT().CreateLogGroup(input).Return(&cloudwatchlogs.CreateLogGroupOutput{}, nil)

e := &Event{
group: testLogGroup,
}

err := output.createLogGroup(e)
assert.NoError(t, err)
}

func TestValidate(t *testing.T) {
for _, test := range configValidationTestCases {
t.Run(test.name, func(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/aws/amazon-kinesis-firehose-for-fluent-bit v1.5.1
github.com/aws/aws-sdk-go v1.44.267
github.com/aws/aws-sdk-go v1.48.4
github.com/fluent/fluent-bit-go v0.0.0-20201210173045-3fd1e0486df2
github.com/golang/mock v1.4.4
github.com/json-iterator/go v1.1.12
Expand Down
24 changes: 2 additions & 22 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
github.com/aws/amazon-kinesis-firehose-for-fluent-bit v1.5.1 h1:6/X+V7X2W2+e2IPCiyhbWhQIMikDkwQ5tPFlcJv2FBk=
github.com/aws/amazon-kinesis-firehose-for-fluent-bit v1.5.1/go.mod h1:alkjOObhCCp4KtT96XPWFi1PRRLUY0teGD8TWgroo2E=
github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.44.267 h1:Asrp6EMqqRxZvjK0NjzkWcrOk15RnWtupuUrUuZMabk=
github.com/aws/aws-sdk-go v1.44.267/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.48.4 h1:HS2L7ynVhkcRrQRro9CLJZ/xLRb4UOzDEfPzgevZwXM=
github.com/aws/aws-sdk-go v1.48.4/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -55,43 +55,23 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
Expand Down