added memory limit config (#2514)

* added memory limit config

* added changelog

* updated changelog

* removed changelog

* Update changelog/fragments/1681906741-memory_limit.yaml

Co-authored-by: Michel Laterman <82832767+michel-laterman@users.noreply.github.com>

* added comment to MemoryLimit

---------

Co-authored-by: Michel Laterman <82832767+michel-laterman@users.noreply.github.com>

Author: juliaElastic

changelog/fragments/1681906741-memory_limit.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Added memory_limit to server.runtime config.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: 2514
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

dev-tools/cloud/docker/build.sh

@@ -11,7 +11,7 @@ REPO_ROOT=$(cd $(dirname $(readlink -f "$0"))/../../.. && pwd)
 USER_NAME=${USER}
 CI_ELASTIC_AGENT_DOCKER_IMAGE=docker.elastic.co/observability-ci/elastic-agent
 
-DEFAULT_IMAGE_TAG=8.8.0-681a23b0-SNAPSHOT
+DEFAULT_IMAGE_TAG=8.8.0-94522507-SNAPSHOT
 BASE_IMAGE="${BASE_IMAGE:-docker.elastic.co/cloud-release/elastic-agent-cloud:$DEFAULT_IMAGE_TAG}"
 GOARCH="${GOARCH:-$(go env GOARCH)}"
 

example/fleet-server-100.yml

@@ -43,6 +43,8 @@ inputs:
         flush_interval: 100ms   # Flush ES bulk queues on this interval.
       runtime:
         gc_percent: 20          # Force the GC to execute more frequently: see https://golang.org/pkg/runtime/debug/#SetGCPercent
+        memory_limit: 4000000000      # Set a soft memory limit to prevent OOM: see https://pkg.go.dev/runtime/debug#SetMemoryLimit 
+
 
 
 http:

internal/pkg/config/runtime.go

@@ -6,6 +6,9 @@ package config
 
 type Runtime struct {
 	GCPercent int `config:"gc_percent"`
+	// set to math.MaxInt64 by default, setting to too low might result in the GC running almost continuously
+	// see https://pkg.go.dev/runtime/debug#SetMemoryLimit
+	MemoryLimit int64 `config:"memory_limit"`
 }
 
 func (r Runtime) InitDefaults() {

internal/pkg/server/fleet.go

@@ -307,6 +307,16 @@ func initRuntime(cfg *config.Config) {
 			Int("new", gcPercent).
 			Msg("SetGCPercent")
 	}
+	memoryLimit := cfg.Inputs[0].Server.Runtime.MemoryLimit
+	if memoryLimit != 0 {
+		old := debug.SetMemoryLimit(memoryLimit)
+
+		log.Info().
+			Int64("old", old).
+			Int64("new", memoryLimit).
+			Msg("SetMemoryLimit")
+	}
+
 }
 
 func (f *Fleet) initBulker(ctx context.Context, tracer *apm.Tracer, cfg *config.Config) (*bulk.Bulker, error) {