[8.x](backport #4318) FIPS compliant builds (#4342)

mergify[bot] · michel-laterman · web-flow · commit bcfe7a57d398 · 2025-01-20T11:38:23.000-08:00
* FIPS compliant builds (#4318) Add FIPS env flag to enable FIPS mode. FIPS=true will change the following: - PLATFORMS will default to linux/amd64 linux/arm64 - make local, make release-* - Binary will be build with -tags=fipsrequired and GOEXPERIMENT=systemcrypto - make build-releaser - chaingaurd microsoft go image will be used as base - make multipass - microsoft's go toolchain will be downloaded and installed to VM (cherry picked from commit f1c29b9) # Conflicts: # Makefile # dev-tools/multipass-cloud-init.yml * Fix merge --------- Co-authored-by: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Co-authored-by: michel-laterman <michel.laterman@elastic.co>
diff --git a/.buildkite/scripts/test-release.sh b/.buildkite/scripts/test-release.sh
@@ -5,6 +5,9 @@ set -euo pipefail
 FLEET_SERVER_VERSION=${1:?"Fleet Server version is needed"}
 
 PLATFORM_FILES=(darwin-aarch64.tar.gz darwin-x86_64.tar.gz linux-arm64.tar.gz linux-x86_64.tar.gz windows-x86_64.zip)
+if [ "$FIPS" = "true" ] ; then
+    PLATFORM_FILES=(linux-arm64-fips.tar.gz linux-x86_64-fips.tar.gz)
+fi
 
 #make release
 
diff --git a/Dockerfile.fips b/Dockerfile.fips
@@ -0,0 +1,45 @@
+ARG GO_VERSION
+FROM mcr.microsoft.com/oss/go/microsoft/golang:${GO_VERSION}-1-fips-bookworm AS base
+
+ENV FIPS=true
+ENV CGO_ENABLED=1
+
+RUN mkdir -p /.cache \
+  && chmod 777 /.cache
+
+WORKDIR /go/src/github.com/elastic/fleet-server
+
+# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+ENTRYPOINT [ "make" ]
+CMD [ "release" ]
+
+FROM base AS builder
+
+COPY . .
+
+ARG GCFLAGS=""
+ARG LDFLAGS=""
+ARG DEV=""
+ARG TARGETPLATFORM
+
+RUN FIPS=true CGO_ENABLED=1 GCFLAGS="${GCFLAGS}" LDFLAGS="${LDFLAGS}" DEV="${DEV}" make release-${TARGETPLATFORM}
+
+FROM docker.elastic.co/wolfi/glibc-openssl-fips:latest
+ARG VERSION
+ARG TARGETOS
+ARG TARGETARCH
+
+RUN groupadd --gid 1000 fleet-server && \
+    useradd -M --uid 1000 --gid 1000 fleet-server
+
+USER fleet-server
+
+COPY --chown=fleet-server:fleet-server --chmod=644 fleet-server.yml /etc/fleet-server.yml
+COPY --chown=fleet-server:fleet-server --chmod=755 --from=builder /usr/src/fleet-server/build/binaries/fleet-server-${VERSION}-${TARGETOS:-linux}-*/fleet-server /usr/bin/fleet-server
+
+ENV GOFIPS=1
+
+CMD /usr/bin/fleet-server -c /etc/fleet-server.yml
diff --git a/Makefile b/Makefile
@@ -62,19 +62,45 @@ GOBIN=$(shell go env GOPATH)/bin/
 
 OS_NAME:=$(shell uname -s)
 
+# Set FIPS=true to force FIPS compliance when building
+FIPS?=
+ifeq "${FIPS}" "true"
+BUILDER_IMAGE=fleet-server-fips-builder:${GO_VERSION}
+PLATFORMS = linux/amd64 linux/arm64
+endif
+
+.EXPORT_ALL_VARIABLES:
+	FIPS=${FIPS}
+
 .PHONY: help
 help: ## - Show help message
 	@printf "${CMD_COLOR_ON} usage: make [target]\n\n${CMD_COLOR_OFF}"
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | sed -e "s/^Makefile://" | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+.PHONY: multipass
+multipass: ## - Launch a multipass instance for development
+ifeq ($(shell uname -p),arm)
+	$(eval ARCH := arm64)
+else
+	$(eval ARCH := amd64)
+endif
+ifeq "${FIPS}" "true"
+	$(eval DOWNLOAD_URL := https://aka.ms/golang/release/latest/go${GO_VERSION}-1.linux-${ARCH}.tar.gz)
+else
+	$(eval DOWNLOAD_URL := https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz)
+endif
+	@cat dev-tools/multipass-cloud-init.yml.envsubst | DOWNLOAD_URL=${DOWNLOAD_URL} ARCH=${ARCH} envsubst > dev-tools/multipass-cloud-init.yml
+	@multipass launch --cloud-init=dev-tools/multipass-cloud-init.yml --mount ..:~/git --name fleet-server-dev --memory 8G --cpus 2 --disk 50G noble
+	@rm dev-tools/multipass-cloud-init.yml
+
 .PHONY: list-platforms
 list-platforms: ## - Show the possible PLATFORMS
 	@echo  "${PLATFORMS}"
 
 .PHONY: local
 local: ## - Build local binary for local environment (bin/fleet-server)
 	@printf "${CMD_COLOR_ON} Build binaries using local go installation\n${CMD_COLOR_OFF}"
-	go build $(if $(SNAPSHOT),-tags="snapshot",) -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o ./bin/fleet-server .
+	$(if $(FIPS),GOEXPERIMENT=systemcrypto) go build $(if $(SNAPSHOT),-tags="snapshot",) $(if $(FIPS),-tags="requirefips",) -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o ./bin/fleet-server .
 	@printf "${CMD_COLOR_ON} Binaries in ./bin/\n${CMD_COLOR_OFF}"
 
 .PHONY: $(COVER_TARGETS)
@@ -84,7 +110,7 @@ $(COVER_TARGETS): cover-%: ## - Build a binary with the -cover flag for integrat
 	$(eval $@_GO_ARCH := $(lastword $(subst /, ,$(lastword $(subst cover-, ,$@)))))
 	$(eval $@_ARCH := $(TARGET_ARCH_$($@_GO_ARCH)))
 	$(eval $@_BUILDMODE:= $(BUILDMODE_$($@_OS)_$($@_GO_ARCH)))
-	GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) go build $(if $(SNAPSHOT),-tags="snapshot",) -cover -coverpkg=./... -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/cover/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)/fleet-server$(if $(filter windows,$($@_OS)),.exe,) .
+	GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) $(if $(FIPS),GOEXPERIMENT=systemcrypto) go build $(if $(SNAPSHOT),-tags="snapshot",) $(if $(FIPS),-tags="requirefips",) -cover -coverpkg=./... -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/cover/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)/fleet-server$(if $(filter windows,$($@_OS)),.exe,) .
 
 .PHONY: clean
 clean: ## - Clean up build artifacts
@@ -159,11 +185,11 @@ test-release:  ## - Check that all release binaries are created
 
 .PHONY: test-unit
 test-unit: prepare-test-context  ## - Run unit tests only
-	set -o pipefail; go test ${GO_TEST_FLAG} -v -race -coverprofile=build/coverage-${OS_NAME}.out ./... | tee build/test-unit-${OS_NAME}.out
+	set -o pipefail; go test ${GO_TEST_FLAG} $(if $(FIPS),-tags="requirefips",) -v -race -coverprofile=build/coverage-${OS_NAME}.out ./... | tee build/test-unit-${OS_NAME}.out
 
 .PHONY: benchmark
 benchmark: prepare-test-context install-benchstat  ## - Run benchmark tests only
-	set -o pipefail; go test -bench=$(BENCHMARK_FILTER) -run=$(BENCHMARK_FILTER) $(BENCHMARK_ARGS) $(BENCHMARK_PACKAGE) | tee "build/$(BENCH_BASE)"
+	set -o pipefail; go test -bench=$(BENCHMARK_FILTER) $(if $(FIPS),-tags="requirefips",) -run=$(BENCHMARK_FILTER) $(BENCHMARK_ARGS) $(BENCHMARK_PACKAGE) | tee "build/$(BENCH_BASE)"
 
 .PHONY: install-benchstat
 install-benchstat: ## - Install the benchstat package
@@ -199,7 +225,7 @@ $(PLATFORM_TARGETS): release-%:
 	$(eval $@_GO_ARCH := $(lastword $(subst /, ,$(lastword $(subst release-, ,$@)))))
 	$(eval $@_ARCH := $(TARGET_ARCH_$($@_GO_ARCH)))
 	$(eval $@_BUILDMODE:= $(BUILDMODE_$($@_OS)_$($@_GO_ARCH)))
-	GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) go build $(if $(SNAPSHOT),-tags="snapshot",) -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/binaries/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)/fleet-server .
+	GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) $(if $(FIPS),GOEXPERIMENT=systemcrypto) go build $(if $(SNAPSHOT),-tags="snapshot",) $(if $(FIPS),-tags="requirefips",) -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/binaries/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)$(if $(FIPS),-fips,)/fleet-server .
 	@$(MAKE) OS=$($@_OS) ARCH=$($@_ARCH) package-target
 
 .PHONY: build-docker
@@ -236,14 +262,14 @@ package-target: build/distributions
 ifeq ($(OS),windows)
 	@mv build/binaries/fleet-server-$(VERSION)-$(OS)-$(ARCH)/fleet-server build/binaries/fleet-server-$(VERSION)-$(OS)-$(ARCH)/fleet-server.exe
 	@cd build/binaries && zip -q -r ../distributions/fleet-server-$(VERSION)-$(OS)-$(ARCH).zip fleet-server-$(VERSION)-$(OS)-$(ARCH)
-	@cd build/distributions && shasum -a 512 fleet-server-$(VERSION)-$(OS)-$(ARCH).zip > fleet-server-$(VERSION)-$(OS)-$(ARCH).zip.sha512
+	@cd build/distributions && sha512sum fleet-server-$(VERSION)-$(OS)-$(ARCH).zip > fleet-server-$(VERSION)-$(OS)-$(ARCH).zip.sha512
 else ifeq ($(OS)-$(ARCH),darwin-arm64)
 	@mv build/binaries/fleet-server-$(VERSION)-$(OS)-$(ARCH) build/binaries/fleet-server-$(VERSION)-$(OS)-aarch64
 	@tar -C build/binaries -zcf build/distributions/fleet-server-$(VERSION)-$(OS)-aarch64.tar.gz fleet-server-$(VERSION)-$(OS)-aarch64
-	@cd build/distributions && shasum -a 512 fleet-server-$(VERSION)-$(OS)-aarch64.tar.gz > fleet-server-$(VERSION)-$(OS)-aarch64.tar.gz.sha512
+	@cd build/distributions && sha512sum fleet-server-$(VERSION)-$(OS)-aarch64.tar.gz > fleet-server-$(VERSION)-$(OS)-aarch64.tar.gz.sha512
 else
-	@tar -C build/binaries -zcf build/distributions/fleet-server-$(VERSION)-$(OS)-$(ARCH).tar.gz fleet-server-$(VERSION)-$(OS)-$(ARCH)
-	@cd build/distributions && shasum -a 512 fleet-server-$(VERSION)-$(OS)-$(ARCH).tar.gz > fleet-server-$(VERSION)-$(OS)-$(ARCH).tar.gz.sha512
+	@tar -C build/binaries -zcf build/distributions/fleet-server-$(VERSION)-$(OS)-$(ARCH)$(if $(FIPS),-fips,).tar.gz fleet-server-$(VERSION)-$(OS)-$(ARCH)$(if $(FIPS),-fips,)
+	@cd build/distributions && sha512sum fleet-server-$(VERSION)-$(OS)-$(ARCH)$(if $(FIPS),-fips,).tar.gz > fleet-server-$(VERSION)-$(OS)-$(ARCH)$(if $(FIPS),-fips,).tar.gz.sha512
 endif
 
 build-releaser: ## - Build a Docker image to run make package including all build tools
@@ -252,16 +278,20 @@ ifeq ($(shell uname -p),arm)
 else
 	$(eval SUFFIX := ${CROSSBUILD_SUFFIX})
 endif
+ifeq "${FIPS}" "true"
+	docker build -t $(BUILDER_IMAGE) -f Dockerfile.fips --target base --build-arg GO_VERSION=$(GO_VERSION) .
+else
 	docker build -t $(BUILDER_IMAGE) -f Dockerfile.build --build-arg GO_VERSION=$(GO_VERSION) --build-arg SUFFIX=${SUFFIX} .
+endif
 
 .PHONY: docker-release
 docker-release: build-releaser ## - Builds a release for all platforms in a dockerised environment
 	docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server $(BUILDER_IMAGE) release
 
 .PHONY: docker-cover-e2e-binaries
 docker-cover-e2e-binaries: build-releaser
-	## Build for local architecture and for linux/amd64 for docker images.
-	docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH) cover-$(shell go env GOOS)/$(shell go env GOARCH)
+	## Build for local architecture and for linux/$ARCH for docker images.
+	docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true $(if $(FIPS),-e FIPS=true) $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH) cover-$(shell go env GOOS)/$(shell go env GOARCH)
 
 .PHONY: release
 release: $(PLATFORM_TARGETS) ## - Builds a release. Specify exact platform with PLATFORMS env.
diff --git a/dev-tools/multipass-cloud-init.yml.envsubst b/dev-tools/multipass-cloud-init.yml.envsubst
@@ -10,7 +10,7 @@
 apt:
   sources:
     docker.list:
-      source: deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable
+      source: deb [arch=${ARCH}] https://download.docker.com/linux/ubuntu noble stable
       keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
 groups:
   - docker
@@ -32,9 +32,12 @@ packages:
   - containerd.io
   - docker-buildx-plugin
   - docker-compose-plugin
+  - unzip
+  - zip
 write_files:
   - content: |
       export PATH=$PATH:/usr/local/go/bin
+      export PATH=$PATH:$(go env GOPATH)/bin
     path: /etc/profile.d/go-bin.sh
   # backwords compatibilityi alias for docker-compose
   # makefile does not use the alias
@@ -43,6 +46,8 @@ write_files:
     path: /etc/profile.d/docker-compose-alias.sh
 runcmd:
   - [ mkdir, /run/go ]
-  - [ wget, https://go.dev/dl/go1.20.4.linux-amd64.tar.gz, -O, /run/go/go.tar.gz ] # FIXME don't hardcode go version
+  - [ wget, ${DOWNLOAD_URL}, -O, /run/go/go.tar.gz ]
   - [ tar, -xzf, /run/go/go.tar.gz, -C, /usr/local/ ]
   - [ rm, -rf, /run/go ]
+  - [ su, ubuntu, -c, "/usr/local/go/bin/go install github.com/magefile/mage@latest" ]
+  - [ su, ubuntu, -c, "/usr/local/go/bin/go install github.com/go-delve/delve/cmd/dlv@latest" ]
diff --git a/docs/fips.md b/docs/fips.md
@@ -0,0 +1,105 @@
+# FIPS support
+
+**NOTE: FIPS Support is in-progress**
+
+The fleet-server can be built in FIPS mode.
+This forces the use of a FIPS compliant provider to handle any cryptographic calls.
+
+Currently FIPS is provided by compiling with the [microsoft/go](https://github.com/microsoft/go) distribution.
+This toolchain must be present for local compilation.
+
+
+## Build changes
+
+As we are using Microsfot/go as a base we follow their conventions.
+
+The buildtag `requirefips` is passed when FIPS is enabled/required.
+Additionally when compiling `GOEXPERIMENT=systemcrypto` is specified.
+
+The `FIPS=true` env var is used by our Makefile as the indicator that controls FIPS.
+This env var is also passed to every child process the Makefile starts.
+The following make commands have different behaviour when FIPS is enabled:
+
+- `make multipass` - Provision a multipass VM with the Microsoft/go toolchain. See [Multipass VM Usage](#multipass-vm-usage) for additional details.
+- `make local` - Compile a fleet-server targetting the machine's GOOS/GOARCH with FIPS enabled
+- `make cover-*` - Compile a coverage and fips enabled fleet-server for e2e tests
+- `make test-unit` - Run unit tests passing the `requirefips` build tag.
+- `make benchmark` - Run benchmarks passing the `requirefips` build tag.
+- `make release-*` - Compile a release binary with FIPS enabled. Will have the name fleet-server-$VERSION-$OS-$ARCH-fips
+- `make package-target` - Will package a FIPS enabled release and produce the sha512 checksum for it.
+- `make build-releaser` - Will create the fleet-server builder image based on Microsoft's FIPS enabled golang image.
+- `make docker-release` - Runs `make release` to produce FIPS enabled binaries in a FIPS docker container.
+- `make docker-cover-e2e-binaries` - Will produce coverage and FIPS enabled binaries for e2e tests from within the same docker container that `build-release` makes
+
+### Multipass VM Usage
+
+A Multipass VM created with `FIPS=true make multipass` is able to compile FIPS enabled golang programs, but is not able to run them.
+When you try to run one the following error occurs:
+```
+GOFIPS=1 ./bin/fleet-server -c fleet-server.yml
+panic: opensslcrypto: can't enable FIPS mode for OpenSSL 3.0.13 30 Jan 2024: openssl: FIPS mode not supported by any provider
+
+goroutine 1 [running]:
+crypto/internal/backend.init.1()
+	/usr/local/go/src/crypto/internal/backend/openssl_linux.go:85 +0x210
+```
+
+In order to be  able to run a FIPS enabled binary, openssl must have a fips provider.
+Openssl [provides instructions on how to do this](https://github.com/openssl/openssl/blob/master/README-FIPS.md).
+
+A TLDR for our multipass container is:
+
+1. Download and compile the FIPS provider for openssl in the VM by running:
+```
+wget https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz
+tar -xzf openssl-3.0.13.tar.gz
+cd openssl-3.0.13
+./Configure enable-fips
+make test
+sudo make install_fips
+sudo openssl fipsinstall -out /usr/local/ssl/fipsmodule.cnf -module /usr/local/lib/ossl-modules/fips.so
+```
+
+2. Copy the `fips.so` module to the system library, in order to find the location run:
+```
+openssl version -m
+```
+
+On my VM I would copy the `fips.so` module with:
+```
+sudo cp /usr/local/lib/ossl-modules/fips.so /usr/lib/aarch64-linux-gnu/ossl-modules/fips.so
+```
+
+3. Create an openssl.cnf for the program to use with the contents:
+```
+config_diagnostics = 1
+openssl_conf = openssl_init
+
+.include /usr/local/ssl/fipsmodule.cnf
+
+[openssl_init]
+providers = provider_sect
+alg_section = algorithm_sect
+
+[provider_sect]
+fips = fips_sect
+base = base_sect
+
+[base_sect]
+activate = 1
+
+[algorithm_sect]
+default_properties = fips=yes
+```
+
+4. Run the program with the `OPENSSL_CONF=openssl.cnf` and `GOFIPS=1` env vars, i.e.,
+```
+OPENSSL_CONF=./openssl.cnf GOFIPS=1 ./bin/fleet-server -c fleet-server.yml
+23:48:47.871 INF Boot fleet-server args=["-c","fleet-server.yml"] commit=55104f6f ecs.version=1.6.0 exe=./bin/fleet-server pid=65037 ppid=5642 service.name=fleet-server service.type=fleet-server version=9.0.0
+i...
+```
+
+## Usage
+
+A FIPS enabled binary should be ran with the env var `GOFIPS=1` set.
+The system/image is required to have a FIPS compliant provider available.
