Wait on the watcher at startup instead of just releasing resources associated with it (#4834)

* Wait on the watcher at startup instead of releasing.

* Add changelog.

* Update changelog/fragments/1717185708-Stop-creating-a-zombie-process-on-each-restart.yaml

Co-authored-by: Blake Rouse <blake.rouse@elastic.co>

---------

Co-authored-by: Blake Rouse <blake.rouse@elastic.co>

Author: cmacknz

changelog/fragments/1717185708-Stop-creating-a-zombie-process-on-each-restart.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Stop creating a zombie process on each restart.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4834
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/agent/application/upgrade/rollback.go

@@ -146,23 +146,24 @@ func InvokeWatcher(log *logger.Logger, agentExecutable string) (*exec.Cmd, error
 	}
 
 	cmd := invokeCmd(agentExecutable)
-	defer func() {
-		if cmd.Process != nil {
-			log.Infof("releasing watcher %v", cmd.Process.Pid)
-			_ = cmd.Process.Release()
-		}
-	}()
-
 	log.Infow("Starting upgrade watcher", "path", cmd.Path, "args", cmd.Args, "env", cmd.Env, "dir", cmd.Dir)
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("failed to start Upgrade Watcher: %w", err)
 	}
 
 	upgradeWatcherPID := cmd.Process.Pid
 	agentPID := os.Getpid()
+
+	go func() {
+		if err := cmd.Wait(); err != nil {
+			log.Infow("Upgrade Watcher exited with error", "agent.upgrade.watcher.process.pid", "agent.process.pid", agentPID, upgradeWatcherPID, "error.message", err)
+		}
+	}()
+
 	log.Infow("Upgrade Watcher invoked", "agent.upgrade.watcher.process.pid", upgradeWatcherPID, "agent.process.pid", agentPID)
 
 	return cmd, nil
+
 }
 
 func restartAgent(ctx context.Context, log *logger.Logger, c client.Client) error {