Add ability to select docker variant. (#5338)

Author: blakerouse

dev-tools/mage/crossbuild.go

@@ -33,6 +33,10 @@ var Platforms = BuildPlatforms.Defaults()
 // are considered to be selected (see isPackageTypeSelected).
 var SelectedPackageTypes []PackageType
 
+// SelectedDockerVariants is the list of docker variants. If empty, all docker variants
+// are considered to be selected (see isDockerVariantSelected).
+var SelectedDockerVariants []DockerVariant
+
 func init() {
 	// Allow overriding via PLATFORMS.
 	if expression := os.Getenv("PLATFORMS"); len(expression) > 0 {
@@ -50,6 +54,18 @@ func init() {
 			SelectedPackageTypes = append(SelectedPackageTypes, p)
 		}
 	}
+
+	// Allow overriding via DOCKER_VARIANTS.
+	if dockerVariants := os.Getenv("DOCKER_VARIANTS"); len(dockerVariants) > 0 {
+		for _, docvariant := range strings.Split(dockerVariants, ",") {
+			var v DockerVariant
+			err := v.UnmarshalText([]byte(docvariant))
+			if err != nil {
+				continue
+			}
+			SelectedDockerVariants = append(SelectedDockerVariants, v)
+		}
+	}
 }
 
 // CrossBuildOption defines an option to the CrossBuild target.

dev-tools/mage/dockerbuilder.go

@@ -28,17 +28,12 @@ type dockerBuilder struct {
 }
 
 func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) {
-	imageName, err := spec.ImageName()
-	if err != nil {
-		return nil, err
-	}
-
 	buildDir := filepath.Join(spec.packageDir, "docker-build")
 	beatDir := filepath.Join(buildDir, "beat")
 
 	return &dockerBuilder{
 		PackageSpec: spec,
-		imageName:   imageName,
+		imageName:   spec.ImageName(),
 		buildDir:    buildDir,
 		beatDir:     beatDir,
 	}, nil
@@ -117,6 +112,7 @@ func (b *dockerBuilder) prepareBuild() error {
 	data := map[string]interface{}{
 		"ExposePorts": b.exposePorts(),
 		"ModulesDirs": b.modulesDirs(),
+		"Variant":     b.DockerVariant.String(),
 	}
 
 	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {

dev-tools/mage/dockervariants.go

@@ -0,0 +1,78 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package mage
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	undefined = "undefined"
+	basic     = "basic"
+	ubi       = "ubi"
+	wolfi     = "wolfi"
+	complete  = "complete"
+	cloud     = "cloud"
+)
+
+// DockerVariant defines the docker variant to build.
+type DockerVariant int
+
+// List of possible docker variants.
+const (
+	Undefined = iota
+	Basic
+	UBI
+	Wolfi
+	Complete
+	Cloud
+)
+
+// String returns the name of the docker variant type.
+func (typ DockerVariant) String() string {
+	switch typ {
+	case Undefined:
+		return undefined
+	case Basic:
+		return basic
+	case UBI:
+		return ubi
+	case Wolfi:
+		return wolfi
+	case Complete:
+		return complete
+	case Cloud:
+		return cloud
+	default:
+		return invalid
+	}
+}
+
+// MarshalText returns the text representation of DockerVariant.
+func (typ DockerVariant) MarshalText() ([]byte, error) {
+	return []byte(typ.String()), nil
+}
+
+// UnmarshalText returns a DockerVariant based on the given text.
+func (typ *DockerVariant) UnmarshalText(text []byte) error {
+	switch strings.ToLower(string(text)) {
+	case "":
+		*typ = Undefined
+	case basic:
+		*typ = Basic
+	case ubi:
+		*typ = UBI
+	case wolfi:
+		*typ = Wolfi
+	case complete:
+		*typ = Complete
+	case cloud:
+		*typ = Cloud
+	default:
+		return fmt.Errorf("unknown docker variant: %v", string(text))
+	}
+	return nil
+}

dev-tools/mage/pkg.go

@@ -47,6 +47,11 @@ func Package() error {
 					continue
 				}
 
+				if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) {
+					log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant)
+					continue
+				}
+
 				if target.Name == "linux/arm64" && pkgType == Docker && runtime.GOARCH != "arm64" {
 					log.Printf("Skipping Docker package type because build host isn't arm")
 					continue
@@ -121,6 +126,21 @@ func isPackageTypeSelected(pkgType PackageType) bool {
 	return false
 }
 
+// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if
+// docVariant is present on SelectedDockerVariants. It returns false otherwise.
+func isDockerVariantSelected(docVariant DockerVariant) bool {
+	if len(SelectedDockerVariants) == 0 {
+		return true
+	}
+
+	for _, v := range SelectedDockerVariants {
+		if v == docVariant {
+			return true
+		}
+	}
+	return false
+}
+
 type packageBuilder struct {
 	Platform BuildPlatform
 	Spec     PackageSpec

dev-tools/mage/pkgspecs.go

@@ -154,5 +154,16 @@ func LoadSpecs(files ...string) (map[string][]OSPackageArgs, error) {
 		return nil, fmt.Errorf("failed to unmarshal spec data: %w", err)
 	}
 
+	// verify that the package specification sets the docker variant
+	for specName, specs := range packages.Specs {
+		for _, spec := range specs {
+			for _, pkgType := range spec.Types {
+				if pkgType == Docker && spec.Spec.DockerVariant == Undefined {
+					return nil, fmt.Errorf("%s defined a package spec for docker without a docker_variant set", specName)
+				}
+			}
+		}
+	}
+
 	return packages.Specs, nil
 }

dev-tools/mage/pkgtypes.go

@@ -91,6 +91,7 @@ type PackageSpec struct {
 	License           string                 `yaml:"license,omitempty"`
 	URL               string                 `yaml:"url,omitempty"`
 	Description       string                 `yaml:"description,omitempty"`
+	DockerVariant     DockerVariant          `yaml:"docker_variant,omitempty"`
 	PreInstallScript  string                 `yaml:"pre_install_script,omitempty"`
 	PostInstallScript string                 `yaml:"post_install_script,omitempty"`
 	PostRmScript      string                 `yaml:"post_rm_script,omitempty"`
@@ -271,11 +272,7 @@ func (typ PackageType) AddFileExtension(file string) string {
 func (typ PackageType) PackagingDir(home string, target BuildPlatform, spec PackageSpec) (string, error) {
 	root := home
 	if typ == Docker {
-		imageName, err := spec.ImageName()
-		if err != nil {
-			return "", err
-		}
-		root = filepath.Join(root, imageName)
+		root = filepath.Join(root, spec.ImageName())
 	}
 
 	targetPath := typ.AddFileExtension(spec.Name + "-" + target.GOOS() + "-" + target.Arch())
@@ -467,17 +464,13 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 	return s
 }
 
-// ImageName computes the image name from the spec. A template for the image
-// name can be configured by adding image_name to extra_vars.
-func (s PackageSpec) ImageName() (string, error) {
-	if name := s.ExtraVars["image_name"]; name != "" {
-		imageName, err := s.Expand(name)
-		if err != nil {
-			return "", fmt.Errorf("failed to expand image_name: %w", err)
-		}
-		return imageName, nil
+// ImageName computes the image name from the spec.
+func (s PackageSpec) ImageName() string {
+	if s.DockerVariant == Basic {
+		// no suffix for basic docker variant
+		return s.Name
 	}
-	return s.Name, nil
+	return fmt.Sprintf("%s-%s", s.Name, s.DockerVariant)
 }
 
 func copyInstallScript(spec PackageSpec, script string, local *string) error {

dev-tools/packaging/packages.yml

@@ -223,31 +223,31 @@ shared:
       buildFrom: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
 
   - &docker_ubuntu_spec
+    docker_variant: 'basic'
     extra_vars:
       from: '--platform=linux/amd64 ubuntu:20.04'
-      image_name: '{{.BeatName}}'
   - &docker_ubuntu_arm_spec
+    docker_variant: 'basic'
     extra_vars:
       from: '--platform=linux/arm64 ubuntu:20.04'
-      image_name: '{{.BeatName}}'
 
   - &docker_ubi_spec
+    docker_variant: 'ubi'
     extra_vars:
       from: '--platform=linux/amd64 docker.elastic.co/ubi9/ubi-minimal'
-      image_name: '{{.BeatName}}-ubi'
   - &docker_ubi_arm_spec
+    docker_variant: 'ubi'
     extra_vars:
       from: '--platform=linux/arm64 docker.elastic.co/ubi9/ubi-minimal'
-      image_name: '{{.BeatName}}-ubi'
 
   - &docker_wolfi_spec
+    docker_variant: 'wolfi'
     extra_vars:
       from: '--platform=linux/amd64 cgr.dev/chainguard/wolfi-base'
-      image_name: '{{.BeatName}}-wolfi'
   - &docker_wolfi_arm_spec
+    docker_variant: 'wolfi'
     extra_vars:
       from: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
-      image_name: '{{.BeatName}}-wolfi'
 
   - &docker_elastic_spec
     extra_vars:
@@ -275,9 +275,10 @@ shared:
           {{ commit }}
         mode: 0644
 
+  # cloud build to beats-ci repository
   - &agent_docker_cloud_spec
+    docker_variant: 'cloud'
     extra_vars:
-      image_name: '{{.BeatName}}-cloud'
       repository: 'docker.elastic.co/beats-ci'
     files:
       'data/cloud_downloads/filebeat.sh':
@@ -290,11 +291,10 @@ shared:
         source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0755
 
-  # not different to the default image, kept for backwards-compatibility
+  # includes nodejs with @elastic/synthetics
   - &agent_docker_complete_spec
     <<: *agent_docker_spec
-    extra_vars:
-      image_name: '{{.BeatName}}-complete'
+    docker_variant: 'complete'
 
   # Deb/RPM spec for community beats.
   - &deb_rpm_spec

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -43,7 +43,7 @@ RUN true && \
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
 
-{{- if contains .image_name "-cloud" }}
+{{- if eq .Variant "cloud" }}
     mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \
     cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \
     chmod +x /opt/filebeat/filebeat && \
@@ -170,14 +170,14 @@ RUN mkdir /licenses
 COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses
 COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 
-{{- if contains .image_name "-cloud" }}
+{{- if eq .Variant "cloud" }}
 COPY --from=home /opt /opt
 # Generate folder for a stub command that will be overwritten at runtime
 RUN mkdir /app && \
     chown {{ .user }}:{{ .user }} /app
 {{- end }}
 
-{{- if (and (contains .image_name "-complete") (contains .from "ubuntu"))  }}
+{{- if (and (eq .Variant "complete") (contains .from "ubuntu"))  }}
 USER root
 ENV NODE_PATH={{ $beatHome }}/.node
 RUN echo \
@@ -254,7 +254,7 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/
 
 WORKDIR {{ $beatHome }}
 
-{{- if contains .image_name "-cloud" }}
+{{- if eq .Variant "cloud" }}
 ENTRYPOINT ["/usr/bin/tini", "--"]
 CMD ["/app/apm.sh"]
 # Generate a stub command that will be overwritten at runtime