update metrics setup

internal/pkg/agent/application/coordinator/coordinator.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/go-multierror"
@@ -285,7 +286,13 @@ type Coordinator struct {
 	// Should only be interacted with via CoordinatorActive() or runLoopIteration()
 	heartbeatChan chan struct{}
 
-	compPidUpdate chan struct{}
+	// if a component (mostly endpoint) has a new PID, we need to update
+	// the monitoring components so they have a PID to monitor
+	// however, if endpoint is in some kind of restart loop,
+	// we could DOS the config system. Instead,
+	// run a ticker that checks to see if we have a new PID.
+	componentPIDTicker         *time.Ticker
+	componentPidRequiresUpdate *atomic.Bool
 }
 
 // The channels Coordinator reads to receive updates from the various managers.
@@ -376,11 +383,12 @@ func New(logger *logger.Logger, cfg *configuration.Configuration, logLevel logp.
 		// synchronization in the subscriber API, just set the input buffer to 0.
 		stateBroadcaster: broadcaster.New(state, 64, 32),
 
-		logLevelCh:         make(chan logp.Level),
-		overrideStateChan:  make(chan *coordinatorOverrideState),
-		upgradeDetailsChan: make(chan *details.Details),
-		heartbeatChan:      make(chan struct{}),
-		compPidUpdate:      make(chan struct{}, 1),
+		logLevelCh:                 make(chan logp.Level),
+		overrideStateChan:          make(chan *coordinatorOverrideState),
+		upgradeDetailsChan:         make(chan *details.Details),
+		heartbeatChan:              make(chan struct{}),
+		componentPIDTicker:         time.NewTicker(time.Second * 30),
+		componentPidRequiresUpdate: &atomic.Bool{},
 	}
 	// Setup communication channels for any non-nil components. This pattern
 	// lets us transparently accept nil managers / simulated events during
@@ -923,6 +931,8 @@ func (c *Coordinator) runner(ctx context.Context) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	defer c.componentPIDTicker.Stop()
+
 	// We run nil checks before starting the various managers so that unit tests
 	// only have to initialize / mock the specific components they're testing.
 	// If a manager is nil, we prebuffer its return channel with nil also so
@@ -1035,12 +1045,17 @@ func (c *Coordinator) runLoopIteration(ctx context.Context) {
 
 	case c.heartbeatChan <- struct{}{}:
 
-	case <-c.compPidUpdate:
-		err := c.refreshComponentModel(ctx)
-		if err != nil {
-			err = fmt.Errorf("error refreshing component model for PID update: %w", err)
-			c.setConfigManagerError(err)
-			c.logger.Errorf("%s", err)
+	case <-c.componentPIDTicker.C:
+		// if we hit the ticker and we've got a new PID,
+		// reload the component model
+		if c.componentPidRequiresUpdate.Load() {
+			c.componentPidRequiresUpdate.Store(false)
+			err := c.refreshComponentModel(ctx)
+			if err != nil {
+				err = fmt.Errorf("error refreshing component model for PID update: %w", err)
+				c.setConfigManagerError(err)
+				c.logger.Errorf("%s", err)
+			}
 		}
 
 	case componentState := <-c.managerChans.runtimeManagerUpdate:

internal/pkg/agent/application/coordinator/coordinator_state.go

@@ -136,6 +136,7 @@ func (c *Coordinator) refreshState() {
 // Must be called on the main Coordinator goroutine.
 func (c *Coordinator) applyComponentState(state runtime.ComponentComponentState) {
 	found := false
+	// check for any component updates to the PID, so we can update the component monitoring
 	pidRequiresUpdate := false
 	for i, other := range c.state.Components {
 		if other.Component.ID == state.Component.ID {
@@ -169,7 +170,7 @@ func (c *Coordinator) applyComponentState(state runtime.ComponentComponentState)
 	c.stateNeedsRefresh = true
 
 	if pidRequiresUpdate {
-		c.compPidUpdate <- struct{}{}
+		c.componentPidRequiresUpdate.Store(true)
 	}
 }
 

internal/pkg/agent/application/monitoring/v1_monitor.go

@@ -910,84 +910,89 @@ func (b *BeatsMonitor) injectMetricsInput(cfg map[string]interface{}, componentI
 		},
 	}
 
+	for _, comp := range componentList {
+		logp.L().Infof("input component %s: %#v", comp.ID, comp.InputSpec.Spec.Service)
+	}
+
 	// add system/process metrics for services that can't be monitored via json/beats metrics
-	// If there's a checkin PID and it contains the "endpoint" string, assume we want to monitor it
-	for id, comp := range existingStateServicePids {
-		logp.L().Infof("component/pid monitoring map is: %#v", existingStateServicePids)
-		if comp != 0 && strings.Contains(id, "endpoint") {
-			logp.L().Infof("creating system/process watcher for pid %d", comp)
-			inputs = append(inputs, map[string]interface{}{
-				idKey:        fmt.Sprintf("%s-endpoint_security", monitoringMetricsUnitID),
-				"name":       fmt.Sprintf("%s-endpoint_security", monitoringMetricsUnitID),
-				"type":       "system/metrics",
-				useOutputKey: monitoringOutput,
-				"data_stream": map[string]interface{}{
-					"namespace": monitoringNamespace,
-				},
-				"streams": []interface{}{
-					map[string]interface{}{
-						idKey: fmt.Sprintf("%s-endpoint_security", monitoringMetricsUnitID),
-						"data_stream": map[string]interface{}{
-							"type":      "metrics",
-							"dataset":   "elastic_agent.endpoint_security",
-							"namespace": monitoringNamespace,
-						},
-						"metricsets":              []interface{}{"process"},
-						"period":                  metricsCollectionIntervalString,
-						"index":                   fmt.Sprintf("metrics-elastic_agent.endpoint_security-%s", monitoringNamespace),
-						"process.pid":             comp,
-						"process.cgroups.enabled": false,
-						"processors": []interface{}{
-							map[string]interface{}{
-								"add_fields": map[string]interface{}{
-									"target": "data_stream",
-									"fields": map[string]interface{}{
-										"type":      "metrics",
-										"dataset":   "elastic_agent.endpoint_security",
-										"namespace": monitoringNamespace,
+	// If there's a checkin PID and the corrisponding component has a service spec section, add a system/process config
+	for _, compState := range componentList {
+		if compState.InputSpec != nil && compState.InputSpec.Spec.Service != nil {
+			if comp, ok := existingStateServicePids[compState.ID]; ok && comp != 0 {
+				inputs = append(inputs, map[string]interface{}{
+					idKey:        fmt.Sprintf("%s-endpoint_security", monitoringMetricsUnitID),
+					"name":       fmt.Sprintf("%s-endpoint_security", monitoringMetricsUnitID),
+					"type":       "system/metrics",
+					useOutputKey: monitoringOutput,
+					"data_stream": map[string]interface{}{
+						"namespace": monitoringNamespace,
+					},
+					"streams": []interface{}{
+						map[string]interface{}{
+							idKey: fmt.Sprintf("%s-endpoint_security", monitoringMetricsUnitID),
+							"data_stream": map[string]interface{}{
+								"type":      "metrics",
+								"dataset":   "elastic_agent.endpoint_security",
+								"namespace": monitoringNamespace,
+							},
+							"metricsets":              []interface{}{"process"},
+							"period":                  metricsCollectionIntervalString,
+							"index":                   fmt.Sprintf("metrics-elastic_agent.endpoint_security-%s", monitoringNamespace),
+							"process.pid":             comp,
+							"process.cgroups.enabled": false,
+							"processors": []interface{}{
+								map[string]interface{}{
+									"add_fields": map[string]interface{}{
+										"target": "data_stream",
+										"fields": map[string]interface{}{
+											"type":      "metrics",
+											"dataset":   "elastic_agent.endpoint_security",
+											"namespace": monitoringNamespace,
+										},
 									},
 								},
-							},
-							map[string]interface{}{
-								"add_fields": map[string]interface{}{
-									"target": "event",
-									"fields": map[string]interface{}{
-										"dataset": "elastic_agent.endpoint_security",
+								map[string]interface{}{
+									"add_fields": map[string]interface{}{
+										"target": "event",
+										"fields": map[string]interface{}{
+											"dataset": "elastic_agent.endpoint_security",
+										},
 									},
 								},
-							},
-							map[string]interface{}{
-								"add_fields": map[string]interface{}{
-									"target": "elastic_agent",
-									"fields": map[string]interface{}{
-										"id":       b.agentInfo.AgentID(),
-										"version":  b.agentInfo.Version(),
-										"snapshot": b.agentInfo.Snapshot(),
-										"process":  "endpoint_security",
+								map[string]interface{}{
+									"add_fields": map[string]interface{}{
+										"target": "elastic_agent",
+										"fields": map[string]interface{}{
+											"id":       b.agentInfo.AgentID(),
+											"version":  b.agentInfo.Version(),
+											"snapshot": b.agentInfo.Snapshot(),
+											"process":  "endpoint_security",
+										},
 									},
 								},
-							},
-							map[string]interface{}{
-								"add_fields": map[string]interface{}{
-									"target": "agent",
-									"fields": map[string]interface{}{
-										"id": b.agentInfo.AgentID(),
+								map[string]interface{}{
+									"add_fields": map[string]interface{}{
+										"target": "agent",
+										"fields": map[string]interface{}{
+											"id": b.agentInfo.AgentID(),
+										},
 									},
 								},
-							},
-							map[string]interface{}{
-								"add_fields": map[string]interface{}{
-									"target": "component",
-									"fields": map[string]interface{}{
-										"binary": "endpoint_security",
-										"id":     id,
+								map[string]interface{}{
+									"add_fields": map[string]interface{}{
+										"target": "component",
+										"fields": map[string]interface{}{
+											"binary": "endpoint_security",
+											"id":     compState.ID,
+										},
 									},
 								},
 							},
 						},
 					},
-				},
-			})
+				})
+			}
+
 		}
 	}
 

internal/pkg/agent/application/monitoring/v1_monitor_test.go

@@ -23,7 +23,6 @@ import (
 )
 
 func TestMonitoringWithEndpoint(t *testing.T) {
-
 	agentInfo, err := info.NewAgentInfo(context.Background(), false)
 	require.NoError(t, err, "Error creating agent info")
 
@@ -59,7 +58,18 @@ func TestMonitoringWithEndpoint(t *testing.T) {
 	// manually declaring all the MonitoringConfig() args since there's a lot of them, and this makes
 	// the test a little more self-describing
 
-	var compList []component.Component
+	compList := []component.Component{
+		{
+			ID: "endpoint-default",
+			InputSpec: &component.InputRuntimeSpec{
+				Spec: component.InputSpec{
+					Service: &component.ServiceSpec{
+						CPort: 7688,
+					},
+				},
+			},
+		},
+	}
 
 	compIdToBinary := map[string]string{
 		"endpoint-default": "endpoint-security",