switch to monotonic clocks for component check-in (#5284)

* switch to monotonic clocks for component check-in

* add comment on monotonic clock stripping

Author: leehinman

changelog/fragments/1723477450-use-monotonic-clock-for-check-in.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: use monotonic clock for component check-in
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5284
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/5277

pkg/component/runtime/command.go

@@ -195,7 +195,12 @@ func (c *commandRuntime) Run(ctx context.Context, comm Communicator) error {
 				// first check-in
 				sendExpected = true
 			}
-			c.lastCheckin = time.Now().UTC()
+			// Warning lastCheckin must contain a
+			// monotonic clock.  Functions like Local(),
+			// UTC(), Round(), AddDate(), etc. remove the
+			// monotonic clock.  See
+			// https://pkg.go.dev/time
+			c.lastCheckin = time.Now()
 			if c.state.syncCheckin(checkin) {
 				changed = true
 			}
@@ -222,7 +227,13 @@ func (c *commandRuntime) Run(ctx context.Context, comm Communicator) error {
 					}
 				} else {
 					// running and should be running
-					now := time.Now().UTC()
+					//
+					// Warning now must contain a
+					// monotonic clock.  Functions like Local(),
+					// UTC(), Round(), AddDate(), etc. remove the
+					// monotonic clock.  See
+					// https://pkg.go.dev/time
+					now := time.Now()
 					if now.Sub(c.lastCheckin) <= checkinPeriod {
 						c.missedCheckins = 0
 					} else {

pkg/component/runtime/service.go

@@ -478,7 +478,11 @@ func (s *serviceRuntime) processCheckin(checkin *proto.CheckinObserved, comm Com
 		// first check-in
 		sendExpected = true
 	}
-	*lastCheckin = time.Now().UTC()
+	// Warning lastCheckin must contain a monotonic clock.
+	// Functions like Local(), UTC(), Round(), AddDate(),
+	// etc. remove the monotonic clock.  See
+	// https://pkg.go.dev/time
+	*lastCheckin = time.Now()
 	if s.state.syncCheckin(checkin) {
 		changed = true
 	}
@@ -505,7 +509,11 @@ func (s *serviceRuntime) isRunning() bool {
 // checkStatus checks check-ins state, called on timer
 func (s *serviceRuntime) checkStatus(checkinPeriod time.Duration, lastCheckin *time.Time, missedCheckins *int) {
 	if s.isRunning() {
-		now := time.Now().UTC()
+		// Warning now must contain a monotonic clock.
+		// Functions like Local(), UTC(), Round(), AddDate(),
+		// etc. remove the monotonic clock.  See
+		// https://pkg.go.dev/time
+		now := time.Now()
 		if lastCheckin.IsZero() {
 			// never checked-in
 			*missedCheckins++