Skip to content

Commit fee49ef

Browse files
michel-latermanmichel-laterman
authored
fleet-server blackbox e2e tests (#2538)
* Add blackbox e2e test suite Add e2e test suite that treats the fleet-server binary as a black box and tests the API. * Add go build coverage flag support * Add e2e tests to jenkinsfile * reset integration/.env * Add review changes * remove username from local testing image * echo docker-compose commands for debugging * use env var for service token * Add log message to test for debugging * two pass linux extraction * fix linux directory creation * fixes * merge macos and linux extraction * Note extracted elastic-agent location * add command work dir, add command to error statement * Fix broken container test * Use multipass for agent install test suite * Disable agetnt install tests * fix tag * Add container log output for failed tests * Use testcontainers for docker test * alter lifecycle * container lifecylce logs to testlog * Alter secret file permissions in container * Copy certs instead of bind-mount * Fix typos * Fix file permissions * Seperate testing requirements from module requirements * Restore original go.mod, fix test-unit target * Use Dockerfile.build to create e2e binaries * Fix target
1 parent 853b513 commit fee49ef

23 files changed

+1435
-9
lines changed

‎.ci/Jenkinsfile

+14
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,20 @@ pipeline {
111111
}
112112
}
113113
}
114+
stage('E2E Test') {
115+
options { skipDefaultCheckout() }
116+
steps {
117+
withGithubNotify(context: "E2E Test", tab: 'tests') {
118+
cleanup()
119+
dir("${BASE_DIR}"){
120+
withGoEnv(){
121+
retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Docker", script: '.ci/scripts/install-docker-compose.sh') }
122+
sh(label: 'test', script: 'make test-e2e')
123+
}
124+
}
125+
}
126+
}
127+
}
114128
stage('Cloud e2e Test') {
115129
options { skipDefaultCheckout() }
116130
steps {

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ dev-tools/cloud/terraform/*.tfvars*
2424
*.swn
2525

2626
.service_token
27+
.kibana_service_token

‎Dockerfile.build

+3-2
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian11
44
RUN \
55
apt-get update \
66
&& apt-get install -y zip \
7-
&& mkdir /.cache \
7+
&& mkdir -p /.cache \
88
&& chmod 777 /.cache
99

1010
WORKDIR /go/src/github.com/elastic/fleet-server
1111

12-
ENTRYPOINT [ "make", "release" ]
12+
ENTRYPOINT [ "make" ]
13+
CMD [ "release" ]

‎Makefile

+66-5
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ BENCHMARK_ARGS := -count=8
2121
BENCHMARK_PACKAGE ?= ./...
2222
BENCHMARK_FILTER ?= Bench
2323

24-
GO_TEST_FLAG = ""
24+
GO_TEST_FLAG =
2525
ifdef TEST_COVERAGE
2626
GO_TEST_FLAG = -covermode=atomic -coverprofile=build/TEST-go-fleet-server-coverage.cov
2727
endif
@@ -44,6 +44,7 @@ DOCKER_IMAGE?=docker.elastic.co/fleet-server/fleet-server
4444

4545

4646
PLATFORM_TARGETS=$(addprefix release-, $(PLATFORMS))
47+
COVER_TARGETS=$(addprefix cover-, $(PLATFORMS))
4748
COMMIT=$(shell git rev-parse --short HEAD)
4849
NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
4950
CMD_COLOR_ON=\033[32m\xE2\x9c\x93
@@ -75,10 +76,23 @@ local: ## - Build local binary for local environment (bin/fleet-server)
7576
go build $(if $(DEV),-tags="dev",) -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o ./bin/fleet-server .
7677
@printf "${CMD_COLOR_ON} Binaries in ./bin/\n${CMD_COLOR_OFF}"
7778

79+
.PHONY: cover-e2e-binaries
80+
cover-e2e-binaries: ## - Build binaries for the test-e2e target with the go 1.20+ cover flag
81+
SNAPSHOT=true $(MAKE) $(COVER_TARGETS)
82+
83+
.PHONY: $(COVER_TARGETS)
84+
$(COVER_TARGETS): cover-%: ## - Build a binary with the -cover flag for integration testing
85+
@mkdir -p build/cover
86+
$(eval $@_OS := $(firstword $(subst /, ,$(lastword $(subst cover-, ,$@)))))
87+
$(eval $@_GO_ARCH := $(lastword $(subst /, ,$(lastword $(subst cover-, ,$@)))))
88+
$(eval $@_ARCH := $(TARGET_ARCH_$($@_GO_ARCH)))
89+
$(eval $@_BUILDMODE:= $(BUILDMODE_$($@_OS)_$($@_GO_ARCH)))
90+
GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) go build $(if $(DEV),-tags="dev",) -cover -coverpkg=./... -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/cover/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)/fleet-server$(if $(filter windows,$($@_OS)),.exe,) .
91+
7892
.PHONY: clean
7993
clean: ## - Clean up build artifacts
8094
@printf "${CMD_COLOR_ON} Clean up build artifacts\n${CMD_COLOR_OFF}"
81-
rm -rf .service_token ./bin/ ./build/
95+
rm -rf .service_token .kibana_service_token ./bin/ ./build/
8296

8397
.PHONY: generate
8498
generate: ## - Generate schema models
@@ -221,7 +235,11 @@ build-releaser: ## - Build a Docker image to run make package including all buil
221235

222236
.PHONY: docker-release
223237
docker-release: build-releaser ## - Builds a release for all platforms in a dockerised environment
224-
docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server $(BUILDER_IMAGE)
238+
docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server $(BUILDER_IMAGE) release
239+
240+
.PHONY: docker-cover-e2e-binaries
241+
docker-cover-e2e-binaries: build-releaser
242+
docker run --rm -u $(shell id -u):$(shell id -g) --volume $(PWD):/go/src/github.com/elastic/fleet-server $(BUILDER_IMAGE) cover-e2e-binaries
225243

226244
.PHONY: release
227245
release: $(PLATFORM_TARGETS) ## - Builds a release. Specify exact platform with PLATFORMS env.
@@ -263,7 +281,7 @@ export $(shell sed 's/=.*//' ./dev-tools/integration/.env)
263281
# Start ES with docker without waiting
264282
.PHONY: int-docker-start-async
265283
int-docker-start-async:
266-
@docker-compose -f ./dev-tools/integration/docker-compose.yml --env-file ./dev-tools/integration/.env up -d --remove-orphans elasticsearch
284+
@docker compose -f ./dev-tools/integration/docker-compose.yml --env-file ./dev-tools/integration/.env up -d --remove-orphans elasticsearch
267285

268286
# Wait for ES to be ready
269287
.PHONY: int-docker-wait
@@ -279,7 +297,8 @@ int-docker-start: ## - Start docker envronment for integration tests and wait un
279297
# Stop integration docker setup
280298
.PHONY: int-docker-stop
281299
int-docker-stop: ## - Stop docker environment for integration tests
282-
@docker-compose -f ./dev-tools/integration/docker-compose.yml --env-file ./dev-tools/integration/.env down
300+
@docker compose -f ./dev-tools/integration/docker-compose.yml --env-file ./dev-tools/integration/.env down
301+
@rm -f .service_token
283302

284303
# Run integration tests with starting/stopping docker
285304
.PHONY: test-int
@@ -300,6 +319,48 @@ test-int-set: ## - Run integration tests without setup
300319
ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} \
301320
go test -v -tags=integration -count=1 -race -p 1 ./...
302321

322+
##################################################
323+
# e2e testing targets
324+
##################################################
325+
326+
# based off build-and-push-cloud-image
327+
.PHONY: build-e2e-agent-image
328+
build-e2e-agent-image: docker-cover-e2e-binaries ## - Build a custom elastic-agent image with fleet-server binaries with coverage enabled injected
329+
@printf "${CMD_COLOR_ON} Creating test e2e agent image\n${CMD_COLOR_OFF}"
330+
GOARCH=amd64 ./dev-tools/e2e/build.sh
331+
332+
.PHONY: e2e-certs
333+
e2e-certs: ## - Use openssl to create a CA, encrypted private key, and signed fleet-server cert testing purposes
334+
@printf "${CMD_COLOR_ON} Creating test e2e certs\n${CMD_COLOR_OFF}"
335+
@./dev-tools/e2e/certs.sh
336+
337+
.PHONY: e2e-docker-start
338+
e2e-docker-start: int-docker-start ## - Start a testing instance of Elasticsearch and Kibana in docker containers
339+
@KIBANA_TOKEN=$(shell ./dev-tools/e2e/get-kibana-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}) docker compose -f ./dev-tools/e2e/docker-compose.yml --env-file ./dev-tools/integration/.env up -d --remove-orphans kibana
340+
@./dev-tools/e2e/wait-for-kibana.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@localhost:5601
341+
342+
.PHONY: e2e-docker-stop
343+
e2e-docker-stop: ## - Tear down testing Elasticsearch and Kibana instances
344+
@KIBANA_TOKEN="supress-warning" docker compose -f ./dev-tools/e2e/docker-compose.yml --env-file ./dev-tools/integration/.env down
345+
rm -f .kibana_service_token
346+
@$(MAKE) int-docker-stop
347+
348+
.PHONY: test-e2e
349+
test-e2e: docker-cover-e2e-binaries build-e2e-agent-image e2e-certs ## - Setup and run the blackbox end to end test suite
350+
@mkdir -p build/e2e-cover
351+
@$(MAKE) e2e-docker-start
352+
@set -o pipefail; $(MAKE) test-e2e-set | tee build/test-e2e.out
353+
@$(MAKE) e2e-docker-stop
354+
355+
.PHONY: test-e2e-set
356+
test-e2e-set: ## - Run the blackbox end to end tests without setup.
357+
cd testing; \
358+
ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}) \
359+
ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} \
360+
AGENT_E2E_IMAGE=$(shell cat "build/e2e-image") \
361+
CGO_ENABLED=1 \
362+
go test -v -tags=e2e -count=1 -race -p 1 ./...
363+
303364
##################################################
304365
# Cloud testing targets
305366
##################################################

‎dev-tools/cloud/docker/build.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22

33
# This script builds an image from the elastic-agent image
4-
# with a locally built apm-server binary injected. Additional
4+
# with a locally built fleet-server binary injected. Additional
55
# flags (e.g. -t <name>) will be passed to `docker build`.
66

77
set -eu
@@ -40,4 +40,4 @@ docker build \
4040
docker push ${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}
4141

4242
echo "Image available at:"
43-
echo "${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}"
43+
echo "${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}"

‎dev-tools/e2e/Dockerfile

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
ARG ELASTIC_AGENT_IMAGE # e.g. docker.elastic.co/cloud-release/elastic-agent-cloud:8.8.0-4671daa2-SNAPSHOT
2+
3+
FROM --platform=linux/amd64 ${ELASTIC_AGENT_IMAGE} as elastic_agent_amd64
4+
ARG STACK_VERSION # e.g. 8.5.0-SNAPSHOT
5+
ARG VCS_REF_SHORT # e.g. abc123
6+
ONBUILD COPY --chmod=0755 --chown=elastic-agent cover/fleet-server-${STACK_VERSION}-linux-x86_64/fleet-server \
7+
./data/elastic-agent-${VCS_REF_SHORT}/components/fleet-server
8+
9+
FROM --platform=linux/arm64 ${ELASTIC_AGENT_IMAGE} as elastic_agent_arm64
10+
ARG STACK_VERSION # e.g. 8.5.0-SNAPSHOT
11+
ARG VCS_REF_SHORT # e.g. abc123
12+
ONBUILD COPY --chmod=0755 --chown=elastic-agent cover/fleet-server-${STACK_VERSION}-linux-arm64/fleet-server \
13+
./data/elastic-agent-${VCS_REF_SHORT}/components/fleet-server
14+
15+
FROM elastic_agent_${TARGETARCH}

‎dev-tools/e2e/build.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
# This script builds an image from the elastic-agent image
4+
# with a locally built fleet-server binary injected. Additional
5+
# flags (e.g. -t <name>) will be passed to `docker build`.
6+
7+
set -eu
8+
9+
REPO_ROOT=$(cd $(dirname $(readlink -f "$0"))/../.. && pwd)
10+
11+
source ${REPO_ROOT}/dev-tools/integration/.env
12+
13+
COMMIT=$(git rev-parse --short HEAD)
14+
CI_ELASTIC_AGENT_DOCKER_IMAGE=docker.elastic.co/observability-ci/elastic-agent
15+
16+
BASE_IMAGE="${BASE_IMAGE:-docker.elastic.co/cloud-release/elastic-agent-cloud:$ELASTICSEARCH_VERSION}"
17+
GOARCH="${GOARCH:-$(go env GOARCH)}"
18+
19+
export DOCKER_BUILDKIT=1
20+
docker pull --platform linux/$GOARCH $BASE_IMAGE
21+
22+
STACK_VERSION=$(docker inspect -f '{{index .Config.Labels "org.label-schema.version"}}' $BASE_IMAGE)
23+
VCS_REF=$(docker inspect -f '{{index .Config.Labels "org.label-schema.vcs-ref"}}' $BASE_IMAGE)
24+
25+
CUSTOM_IMAGE_TAG=${STACK_VERSION}-e2e-${COMMIT}-$(date +%s)
26+
27+
docker build \
28+
-f $REPO_ROOT/dev-tools/e2e/Dockerfile \
29+
--build-arg ELASTIC_AGENT_IMAGE=$BASE_IMAGE \
30+
--build-arg STACK_VERSION=$STACK_VERSION \
31+
--build-arg VCS_REF_SHORT=${VCS_REF:0:6} \
32+
--platform linux/$GOARCH \
33+
-t ${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG} \
34+
$* $REPO_ROOT/build
35+
36+
echo "${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}" > ${REPO_ROOT}/build/e2e-image

‎dev-tools/e2e/certs.sh

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
3+
set -eu
4+
5+
REPO_ROOT=$(cd $(dirname $(readlink -f "$0"))/../.. && pwd)
6+
CERT_DIR=${REPO_ROOT}/build/e2e-certs
7+
8+
mkdir -p ${CERT_DIR}
9+
10+
# Create CA
11+
openssl req -x509 \
12+
-sha256 -days 356 \
13+
-nodes \
14+
-newkey rsa:2048 \
15+
-subj "/CN=e2e-test-ca" \
16+
-keyout ${CERT_DIR}/e2e-test-ca.key -out ${CERT_DIR}/e2e-test-ca.crt \
17+
2>/dev/null
18+
19+
# Make encrypted private key
20+
echo -n abcd1234 > ${CERT_DIR}/passphrase
21+
22+
openssl genpkey -algorithm RSA \
23+
-aes-128-cbc \
24+
-pkeyopt rsa_keygen_bits:2048 \
25+
-pass file:${CERT_DIR}/passphrase \
26+
-out ${CERT_DIR}/fleet-server-key \
27+
2>/dev/null
28+
29+
openssl rsa -aes-128-cbc \
30+
-in ${CERT_DIR}/fleet-server-key \
31+
-out ${CERT_DIR}/fleet-server.key \
32+
-passin pass:abcd1234 \
33+
-passout file:${CERT_DIR}/passphrase \
34+
2>/dev/null
35+
36+
# Make CSR
37+
openssl req -new \
38+
-key ${CERT_DIR}/fleet-server.key \
39+
-passin file:${CERT_DIR}/passphrase \
40+
-subj "/CN=localhost" \
41+
-addext "subjectAltName=IP:127.0.0.1,DNS:localhost" \
42+
-out ${CERT_DIR}/fleet-server.csr \
43+
2>/dev/null
44+
45+
# Sign CSR with CA
46+
openssl x509 -req \
47+
-in ${CERT_DIR}/fleet-server.csr \
48+
-days 356 \
49+
-extfile <(printf "subjectAltName=IP:127.0.0.1,DNS:localhost") \
50+
-CA ${CERT_DIR}/e2e-test-ca.crt \
51+
-CAkey ${CERT_DIR}/e2e-test-ca.key \
52+
-CAcreateserial \
53+
-out ${CERT_DIR}/fleet-server.crt \
54+
2>/dev/null
55+
56+
# Sanity checks
57+
openssl verify -verbose \
58+
-CAfile ${CERT_DIR}/e2e-test-ca.crt \
59+
${CERT_DIR}/fleet-server.crt
60+
61+
openssl rsa -check -noout \
62+
-in ${CERT_DIR}/fleet-server.key \
63+
-passin file:${CERT_DIR}/passphrase

‎dev-tools/e2e/docker-compose.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
version: '2.3'
2+
services:
3+
kibana:
4+
image: "docker.elastic.co/kibana/kibana:${ELASTICSEARCH_VERSION}-amd64"
5+
container_name: kibana
6+
environment:
7+
- KIBANA_TOKEN=${KIBANA_TOKEN}
8+
volumes:
9+
- ./kibana.yml:/usr/share/kibana/config/kibana.yml
10+
ports:
11+
- 127.0.0.1:5601:5601
12+
networks:
13+
- integration
14+
# Attach to the integration test network
15+
networks:
16+
integration:
17+
name: integration_default
18+
external: true

‎dev-tools/e2e/get-kibana-servicetoken.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
host="$1"
4+
5+
jsonBody="$(curl -sSL -XPOST "$host/_security/service/elastic/kibana/credential/token/token1")"
6+
7+
# use grep and sed to get the service token value as we may not have jq or a similar tool on the instance
8+
token=$(echo ${jsonBody} | grep -Eo '"value"[^}]*' | grep -Eo ':.*' | sed -r "s/://" | sed -r 's/"//g')
9+
10+
# cache or use cached token in order to be able to run repeative integration tests,
11+
# very useful during development, without recreating elasticsearch instance every time.
12+
if [ -z "$token" ]
13+
then
14+
token=`cat .kibana_service_token`
15+
else
16+
echo "$token" > .kibana_service_token
17+
fi
18+
19+
echo "$token"

‎dev-tools/e2e/kibana.yml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
server.name: kibana
2+
server.host: "0.0.0.0"
3+
server.ssl.enabled: false
4+
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
5+
elasticsearch.serviceAccountToken: "${KIBANA_TOKEN}"
6+
7+
xpack.fleet.agents.enabled: true
8+
xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"]
9+
xpack.fleet.agents.elasticsearch.hosts: ["http://elasticsearch:9200"]
10+
xpack.fleet.packages:
11+
- name: elastic_agent
12+
version: latest
13+
- name: fleet_server
14+
version: latest
15+
xpack.fleet.agentPolicies:
16+
- name: fleet-server-policy
17+
id: fleet-server-policy
18+
monitoring_enabled: []
19+
is_default_fleet_server: true
20+
is_managed: false
21+
namespace: default
22+
package_policies:
23+
- name: fleet_server-1
24+
id: default-fleet-server
25+
package:
26+
name: fleet_server
27+
xpack.fleet.outputs:
28+
- id: fleet-default-output
29+
name: default
30+
type: elasticsearch
31+
hosts: [ http://elasticsearch:9200 ]
32+
is_default: true
33+
is_default_monitoring: true

‎dev-tools/e2e/wait-for-kibana.sh

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
host="$1"
6+
shift
7+
cmd="$@"
8+
9+
10+
until $(curl --output /dev/null --silent --head --fail "${host}/api/status"); do
11+
printf '.'
12+
sleep 1
13+
done
14+
15+
# First wait for ES to start...
16+
response=$(curl $host)
17+
18+
until [ "$response" = "200" ]; do
19+
response=$(curl --write-out %{http_code} --silent --output /dev/null "${host}/api/status")
20+
echo '.'
21+
sleep 1
22+
done
23+
24+
>&2 echo "Kibana is up"
25+
exec $cmd

0 commit comments

Comments
 (0)