Skip to content

Commit 46a4d48

Browse files
kubasobonkubasobon
authored
Add correct host info for cloud provider virtual machines (#2126)
1 parent 6585138 commit 46a4d48

File tree

12 files changed

+278
-8
lines changed

12 files changed

+278
-8
lines changed

‎_meta/config/processors.yml.tmpl

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
# Configure processors to enhance or manipulate events generated by the beat.
44

55
processors:
6-
- add_host_metadata: ~
6+
# in case you run in EKS/Kubernetes environment, you can use the following processor to add node metadata
7+
# - add_host_metadata: ~
78
- add_cloud_metadata: ~
89
- add_docker_metadata: ~
910
# in case you run in EKS/Kubernetes environment, you can use the following processor to add cluster id

‎cloudbeat.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,8 @@ output.elasticsearch:
117117
# Configure processors to enhance or manipulate events generated by the beat.
118118

119119
processors:
120-
- add_host_metadata: ~
120+
# in case you run in EKS/Kubernetes environment, you can use the following processor to add node metadata
121+
# - add_host_metadata: ~
121122
- add_cloud_metadata: ~
122123
- add_docker_metadata: ~
123124
# in case you run in EKS/Kubernetes environment, you can use the following processor to add cluster id

‎cmd/root.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/elastic/beats/v7/libbeat/cmd"
2424
"github.com/elastic/beats/v7/libbeat/cmd/instance"
2525
"github.com/elastic/beats/v7/libbeat/common/reload"
26+
"github.com/elastic/beats/v7/libbeat/publisher/processing"
2627
_ "github.com/elastic/beats/v7/x-pack/libbeat/include"
2728
"github.com/elastic/beats/v7/x-pack/libbeat/management"
2829
"github.com/elastic/elastic-agent-client/v7/pkg/client"
@@ -36,7 +37,16 @@ import (
3637
var Name = "cloudbeat"
3738

3839
// RootCmd to handle beats cli
39-
var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, Version: version.CloudbeatSemanticVersion()})
40+
var RootCmd = cmd.GenRootCmdWithSettings(
41+
beater.New,
42+
instance.Settings{
43+
Name: Name,
44+
Version: version.CloudbeatSemanticVersion(),
45+
// Supply our own processing pipeline. Same as processing.MakeDefaultBeatSupport, but without
46+
// `processing.WithHost`.
47+
Processing: processing.MakeDefaultSupport(true, nil, processing.WithECS, processing.WithAgentMeta()),
48+
},
49+
)
4050

4151
func cloudbeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) {
4252
modules, err := management.CreateInputsFromStreams(rawIn, "logs", agentInfo)

‎deploy/aws/cloudbeat-aws.yml

-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ output.elasticsearch:
4747

4848
# ================================= Processors =================================
4949
processors:
50-
- add_host_metadata: ~
5150
- add_cloud_metadata: ~
5251
- add_docker_metadata: ~
5352

‎deploy/azure/cloudbeat-azure.yml

-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ output.elasticsearch:
4949

5050
# ================================= Processors =================================
5151
processors:
52-
- add_host_metadata: ~
5352
- add_cloud_metadata: ~
5453
- add_docker_metadata: ~
5554

‎deploy/gcp/cloudbeat-gcp.yml

-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ output.elasticsearch:
4949

5050
# ================================= Processors =================================
5151
processors:
52-
- add_host_metadata: ~
5352
- add_cloud_metadata: ~
5453
- add_docker_metadata: ~
5554

‎deploy/kustomize/overlays/cloudbeat-aws/cloudbeat.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ output.elasticsearch:
4848

4949
# ================================= Processors =================================
5050
processors:
51-
- add_host_metadata: ~
51+
# - add_host_metadata: ~
5252
- add_cloud_metadata: ~
5353
- add_docker_metadata: ~
5454
- add_cluster_id: ~

‎internal/flavors/posture.go

+21
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ func newPostureFromCfg(b *beat.Beat, cfg *config.Config) (*posture, error) {
6464
return nil, err
6565
}
6666

67+
err = ensureHostProcessor(log, cfg)
68+
if err != nil {
69+
cancel()
70+
return nil, err
71+
}
72+
6773
client, err := NewClient(b.Publisher, cfg.Processors)
6874
if err != nil {
6975
cancel()
@@ -108,3 +114,18 @@ func (bt *posture) Stop() {
108114

109115
bt.cancel()
110116
}
117+
118+
// ensureAdditionalProcessors modifies cfg.Processors list to ensure 'host'
119+
// processor is present for K8s and EKS benchmarks.
120+
func ensureHostProcessor(log *logp.Logger, cfg *config.Config) error {
121+
if cfg.Benchmark != config.CIS_EKS && cfg.Benchmark != config.CIS_K8S {
122+
return nil
123+
}
124+
log.Info("Adding host processor config")
125+
hostProcessor, err := agentconfig.NewConfigFrom("add_host_metadata: ~")
126+
if err != nil {
127+
return err
128+
}
129+
cfg.Processors = append(cfg.Processors, hostProcessor)
130+
return nil
131+
}

‎internal/resources/fetching/fetchers/azure/assets_fetcher.go

+25
Original file line numberDiff line numberDiff line change
@@ -175,5 +175,30 @@ func (r *AzureResource) GetMetadata() (fetching.ResourceMetadata, error) {
175175
}
176176

177177
func (r *AzureResource) GetElasticCommonData() (map[string]any, error) {
178+
if r.Asset.Type == inventory.VirtualMachineAssetType {
179+
m := map[string]any{
180+
"host.name": r.Asset.Name,
181+
}
182+
183+
// "host.hostname" = "properties.osProfile.computerName" if it exists
184+
osProfileRaw, ok := r.Asset.Properties["osProfile"]
185+
if !ok {
186+
return m, nil
187+
}
188+
osProfile, ok := osProfileRaw.(map[string]any)
189+
if !ok {
190+
return m, nil
191+
}
192+
computerNameRaw, ok := osProfile["computerName"]
193+
if !ok {
194+
return m, nil
195+
}
196+
computerName, ok := computerNameRaw.(string)
197+
if !ok {
198+
return m, nil
199+
}
200+
m["host.hostname"] = computerName
201+
return m, nil
202+
}
178203
return nil, nil
179204
}

‎internal/resources/fetching/fetchers/azure/assets_fetcher_test.go

+142-1
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,11 @@ func (s *AzureAssetsFetcherTestSuite) TestFetcher_Fetch() {
220220

221221
ecs, err := result.GetElasticCommonData()
222222
s.Require().NoError(err)
223-
s.Empty(ecs)
223+
if expected.Type == inventory.VirtualMachineAssetType {
224+
s.Contains(ecs, "host.name")
225+
} else {
226+
s.Empty(ecs)
227+
}
224228
})
225229
}
226230
}
@@ -266,6 +270,143 @@ func (s *AzureAssetsFetcherTestSuite) TestFetcher_Fetch_Errors() {
266270
}, metadata)
267271
}
268272

273+
func (s *AzureAssetsFetcherTestSuite) TestFetcher_Fetch_VM_Hostname_Empty() {
274+
mockAssetGroups := make(map[string][]inventory.AzureAsset)
275+
totalMockAssets := 0
276+
var flatMockAssets []inventory.AzureAsset
277+
for _, assetGroup := range AzureAssetGroups {
278+
var mockAssets []inventory.AzureAsset
279+
propertyVariations := [](map[string]any){
280+
map[string]any{"definitelyNotOsProfile": "nope"},
281+
map[string]any{"osProfile": map[string]any{"notComputerName": "nope"}},
282+
map[string]any{"osProfile": map[string]any{"computerName": 42}},
283+
}
284+
for _, properties := range propertyVariations {
285+
mockAssets = append(mockAssets,
286+
inventory.AzureAsset{
287+
Id: "id",
288+
Name: "name",
289+
Location: "location",
290+
Properties: properties,
291+
ResourceGroup: "rg",
292+
SubscriptionId: "subId",
293+
TenantId: "tenantId",
294+
Type: inventory.VirtualMachineAssetType,
295+
Sku: map[string]any{"key": "value"},
296+
Identity: map[string]any{"key": "value"},
297+
},
298+
)
299+
}
300+
totalMockAssets += len(mockAssets)
301+
mockAssetGroups[assetGroup] = mockAssets
302+
flatMockAssets = append(flatMockAssets, mockAssets...)
303+
}
304+
305+
mockProvider := azurelib.NewMockProviderAPI(s.T())
306+
mockProvider.EXPECT().GetSubscriptions(mock.Anything, mock.Anything).Return(
307+
map[string]governance.Subscription{
308+
"subId": {
309+
FullyQualifiedID: "subId",
310+
ShortID: "subId",
311+
DisplayName: "subName",
312+
ManagementGroup: governance.ManagementGroup{
313+
FullyQualifiedID: "mgId",
314+
DisplayName: "mgName",
315+
},
316+
},
317+
}, nil,
318+
).Twice()
319+
mockProvider.EXPECT().
320+
ListDiagnosticSettingsAssetTypes(mock.Anything, cycle.Metadata{}, []string{"subId"}).
321+
Return(nil, nil).
322+
Once()
323+
mockProvider.EXPECT().
324+
ListAllAssetTypesByName(mock.Anything, mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).
325+
RunAndReturn(func(_ context.Context, assetGroup string, _ []string) ([]inventory.AzureAsset, error) {
326+
return mockAssetGroups[assetGroup], nil
327+
})
328+
329+
results, err := s.fetch(mockProvider, totalMockAssets)
330+
s.Require().NoError(err)
331+
for index, result := range results {
332+
expected := flatMockAssets[index]
333+
s.Run(expected.Type, func() {
334+
s.Equal(expected, result.GetData())
335+
336+
ecs, err := result.GetElasticCommonData()
337+
s.Require().NoError(err)
338+
s.Contains(ecs, "host.name")
339+
s.NotContains(ecs, "host.hostname", "ECS data should not contain host.hostname for incomplete properties")
340+
})
341+
}
342+
}
343+
344+
func (s *AzureAssetsFetcherTestSuite) TestFetcher_Fetch_VM_Hostname_OK() {
345+
mockAssetGroups := make(map[string][]inventory.AzureAsset)
346+
totalMockAssets := 0
347+
var flatMockAssets []inventory.AzureAsset
348+
for _, assetGroup := range AzureAssetGroups {
349+
var mockAssets []inventory.AzureAsset
350+
mockAssets = append(mockAssets,
351+
inventory.AzureAsset{
352+
Id: "id",
353+
Name: "name",
354+
Location: "location",
355+
Properties: map[string]any{"osProfile": map[string]any{"computerName": "hostname"}},
356+
ResourceGroup: "rg",
357+
SubscriptionId: "subId",
358+
TenantId: "tenantId",
359+
Type: inventory.VirtualMachineAssetType,
360+
Sku: map[string]any{"key": "value"},
361+
Identity: map[string]any{"key": "value"},
362+
},
363+
)
364+
totalMockAssets += len(mockAssets)
365+
mockAssetGroups[assetGroup] = mockAssets
366+
flatMockAssets = append(flatMockAssets, mockAssets...)
367+
}
368+
369+
mockProvider := azurelib.NewMockProviderAPI(s.T())
370+
mockProvider.EXPECT().GetSubscriptions(mock.Anything, mock.Anything).Return(
371+
map[string]governance.Subscription{
372+
"subId": {
373+
FullyQualifiedID: "subId",
374+
ShortID: "subId",
375+
DisplayName: "subName",
376+
ManagementGroup: governance.ManagementGroup{
377+
FullyQualifiedID: "mgId",
378+
DisplayName: "mgName",
379+
},
380+
},
381+
}, nil,
382+
).Twice()
383+
mockProvider.EXPECT().
384+
ListDiagnosticSettingsAssetTypes(mock.Anything, cycle.Metadata{}, []string{"subId"}).
385+
Return(nil, nil).
386+
Once()
387+
mockProvider.EXPECT().
388+
ListAllAssetTypesByName(mock.Anything, mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).
389+
RunAndReturn(func(_ context.Context, assetGroup string, _ []string) ([]inventory.AzureAsset, error) {
390+
return mockAssetGroups[assetGroup], nil
391+
})
392+
393+
results, err := s.fetch(mockProvider, totalMockAssets)
394+
s.Require().NoError(err)
395+
for index, result := range results {
396+
expected := flatMockAssets[index]
397+
s.Run(expected.Type, func() {
398+
s.Equal(expected, result.GetData())
399+
400+
ecs, err := result.GetElasticCommonData()
401+
s.Require().NoError(err)
402+
s.Contains(ecs, "host.name")
403+
s.Equal("name", ecs["host.name"])
404+
s.Contains(ecs, "host.hostname")
405+
s.Equal("hostname", ecs["host.hostname"])
406+
})
407+
}
408+
}
409+
269410
func (s *AzureAssetsFetcherTestSuite) fetch(provider *azurelib.MockProviderAPI, expectedLength int) ([]fetching.ResourceInfo, error) {
270411
fetcher := AzureAssetsFetcher{
271412
log: testhelper.NewLogger(s.T()),

‎internal/resources/fetching/fetchers/gcp/assets_fetcher.go

+20
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,26 @@ func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {
148148
}
149149

150150
func (r *GcpAsset) GetElasticCommonData() (map[string]any, error) {
151+
if r.Type == fetching.CloudCompute && r.SubType == inventory.ComputeInstanceAssetType {
152+
m := map[string]any{}
153+
data := r.ExtendedAsset.Resource.Data
154+
if data == nil {
155+
return nil, nil
156+
}
157+
nameField, ok := data.Fields["name"]
158+
if ok {
159+
if name := nameField.GetStringValue(); name != "" {
160+
m["host.name"] = name
161+
}
162+
}
163+
hostnameField, ok := data.Fields["hostname"]
164+
if ok {
165+
if hostname := hostnameField.GetStringValue(); hostname != "" {
166+
m["host.hostname"] = hostname
167+
}
168+
}
169+
return m, nil
170+
}
151171
return nil, nil
152172
}
153173

‎internal/resources/fetching/fetchers/gcp/assets_fetcher_test.go

+54
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/samber/lo"
2626
"github.com/stretchr/testify/mock"
2727
"github.com/stretchr/testify/suite"
28+
"google.golang.org/protobuf/types/known/structpb"
2829

2930
"github.com/elastic/cloudbeat/internal/resources/fetching"
3031
"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
@@ -96,3 +97,56 @@ func (s *GcpAssetsFetcherTestSuite) TestFetcher_Fetch() {
9697
s.Equal("orgName", cloudAccountMetadata.OrganizationName)
9798
})
9899
}
100+
101+
func (s *GcpAssetsFetcherTestSuite) TestFetcher_ElasticCommonData() {
102+
cases := []struct {
103+
resourceData map[string]any
104+
expectedECS map[string]any
105+
}{
106+
{
107+
resourceData: map[string]any{},
108+
expectedECS: map[string]any{},
109+
},
110+
{
111+
resourceData: map[string]any{"name": ""},
112+
expectedECS: map[string]any{},
113+
},
114+
{
115+
resourceData: map[string]any{"name": "henrys-vm"},
116+
expectedECS: map[string]any{"host.name": "henrys-vm"},
117+
},
118+
{
119+
resourceData: map[string]any{"hostname": ""},
120+
expectedECS: map[string]any{},
121+
},
122+
{
123+
resourceData: map[string]any{"hostname": "henrys-vm"},
124+
expectedECS: map[string]any{"host.hostname": "henrys-vm"},
125+
},
126+
{
127+
resourceData: map[string]any{"name": "x", "hostname": "y"},
128+
expectedECS: map[string]any{"host.name": "x", "host.hostname": "y"},
129+
},
130+
}
131+
132+
for _, tc := range cases {
133+
dataStruct, err := structpb.NewStruct(tc.resourceData)
134+
s.Require().NoError(err)
135+
136+
asset := &GcpAsset{
137+
Type: fetching.CloudCompute,
138+
SubType: inventory.ComputeInstanceAssetType,
139+
ExtendedAsset: &inventory.ExtendedGcpAsset{
140+
Asset: &assetpb.Asset{
141+
Resource: &assetpb.Resource{
142+
Data: dataStruct,
143+
},
144+
},
145+
},
146+
}
147+
148+
ecs, err := asset.GetElasticCommonData()
149+
s.Require().NoError(err)
150+
s.Equal(tc.expectedECS, ecs)
151+
}
152+
}

0 commit comments

Comments
 (0)