Skip to content

Commit 46baf56

Browse files
pchilabelimawr
pchila
and
belimawr
authored
Use skaffold to run agent in k8s (#2738)
* Build agent docker image and deploy with skaffold * add docs * Update docs/local-k8s-testing.md Co-authored-by: Tiago Queiroz <contato@tiago.eti.br> --------- Co-authored-by: Tiago Queiroz <contato@tiago.eti.br>
1 parent c4748ae commit 46baf56

11 files changed

+348
-0
lines changed

‎.dockerignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
build/
2+
deploy/
3+
.ogc_cache/
4+
.agent_testing/
5+
data/

‎Dockerfile.skaffold

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
ARG GO_VERSION=1.19.9
2+
ARG crossbuild_image="docker.elastic.co/beats-dev/golang-crossbuild"
3+
ARG AGENT_VERSION=8.9.0-SNAPSHOT
4+
ARG AGENT_IMAGE="docker.elastic.co/beats/elastic-agent"
5+
6+
7+
FROM ${crossbuild_image}:${GO_VERSION}-main-debian8 as build
8+
ARG DEV="true"
9+
ARG SNAPSHOT="true"
10+
RUN go install github.com/magefile/mage@v1.15.0
11+
WORKDIR /elastic-agent/
12+
COPY go.mod go.sum /elastic-agent/
13+
RUN go mod download
14+
COPY . /elastic-agent/
15+
RUN mage golangcrossBuild
16+
RUN git rev-parse HEAD | cut -c 1-6 > .build_hash.txt
17+
18+
19+
FROM ${AGENT_IMAGE}:${AGENT_VERSION}
20+
21+
22+
COPY --from=build /elastic-agent/build/golang-crossbuild/elastic-agent-linux-amd64 /usr/share/elastic-agent/elastic-agent
23+
COPY --from=build /elastic-agent/.build_hash.txt /usr/share/elastic-agent/.build_hash.txt.new
24+
RUN mv /usr/share/elastic-agent/data/elastic-agent-$(cat /usr/share/elastic-agent/.build_hash.txt| cut -c 1-6) /usr/share/elastic-agent/data/elastic-agent-$(cat /usr/share/elastic-agent/.build_hash.txt.new| cut -c 1-6) && \
25+
ln -s -f /usr/share/elastic-agent/data/elastic-agent-$(cat /usr/share/elastic-agent/.build_hash.txt.new| cut -c 1-6)/elastic-agent /usr/share/elastic-agent/elastic-agent &&\
26+
mv /usr/share/elastic-agent/.build_hash.txt /usr/share/elastic-agent/.build_hash.txt.old && \
27+
mv /usr/share/elastic-agent/.build_hash.txt.new /usr/share/elastic-agent/.build_hash.txt

‎deploy/skaffold/.env.example

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# insert your connection parameters here and save it as .env before running skaffold commands
2+
3+
# managed elastic agent vars
4+
FLEET_URL=https://<fleet host>:443
5+
FLEET_ENROLLMENT_TOKEN=<enrollment token>
6+
7+
# standalone elastic agent vars
8+
ES_HOST=https://<elasticsearch host>:443
9+
ES_USERNAME=elastic
10+
ES_PASSWORD=changeme

‎deploy/skaffold/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
elastic-agent-kubernetes.yaml
2+
.env

‎deploy/skaffold/kustomization.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
5+
resources:
6+
- elastic-agent-kubernetes.yaml
7+
8+
patches:
9+
- path: patches/elastic-agent-standalone-entrypoint.yaml
10+
target:
11+
group: apps
12+
version: v1
13+
kind: DaemonSet
14+
name: elastic-agent-standalone
15+
16+
- path: patches/elastic-agent-managed-entrypoint.yaml
17+
target:
18+
group: apps
19+
version: v1
20+
kind: DaemonSet
21+
name: elastic-agent
22+
23+
- path: patches/fleet-env-vars.yaml
24+
target:
25+
group: apps
26+
version: v1
27+
kind: DaemonSet
28+
name: elastic-agent
29+
30+
- path: patches/elasticsearch-env-vars.yaml
31+
target:
32+
group: apps
33+
version: v1
34+
kind: DaemonSet
35+
name: elastic-agent-standalone
36+
37+
configMapGenerator:
38+
- name: fleet-es-configmap
39+
namespace: kube-system
40+
envs:
41+
- .env

‎deploy/skaffold/patches/elastic-agent-managed-entrypoint.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: elastic-agent
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: elastic-agent
10+
command:
11+
- /usr/share/elastic-agent/elastic-agent
12+
args: ["container", "-e"]

‎deploy/skaffold/patches/elastic-agent-standalone-entrypoint.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: elastic-agent-standalone
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: elastic-agent-standalone
10+
command:
11+
- /usr/share/elastic-agent/elastic-agent
12+
args: ["container", "-c", "/etc/elastic-agent/agent.yml", "-e"]

‎deploy/skaffold/patches/elasticsearch-env-vars.yaml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: elastic-agent-standalone
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: elastic-agent-standalone
10+
env:
11+
- name: ES_USERNAME
12+
value: ""
13+
valueFrom:
14+
configMapKeyRef:
15+
name: fleet-es-configmap
16+
key: ES_USERNAME
17+
# The basic authentication password used to connect to Elasticsearch
18+
- name: ES_PASSWORD
19+
value: ""
20+
valueFrom:
21+
configMapKeyRef:
22+
name: fleet-es-configmap
23+
key: ES_PASSWORD
24+
# The Elasticsearch host to communicate with
25+
- name: ES_HOST
26+
value: ""
27+
valueFrom:
28+
configMapKeyRef:
29+
name: fleet-es-configmap
30+
key: ES_HOST

‎deploy/skaffold/patches/fleet-env-vars.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: elastic-agent
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: elastic-agent
10+
env:
11+
- name: FLEET_URL
12+
value: ""
13+
valueFrom:
14+
configMapKeyRef:
15+
name: fleet-es-configmap
16+
key: FLEET_URL
17+
# Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens)
18+
# If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed
19+
- name: FLEET_ENROLLMENT_TOKEN
20+
value: ""
21+
valueFrom:
22+
configMapKeyRef:
23+
name: fleet-es-configmap
24+
key: FLEET_ENROLLMENT_TOKEN
25+
# - name: KIBANA_HOST
26+
# value: "$(KIBANA_HOST)"
27+
# # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet
28+
# - name: KIBANA_FLEET_USERNAME
29+
# value: "$(KIBANA_FLEET_USERNAME)"
30+
# # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet
31+
# - name: KIBANA_FLEET_PASSWORD
32+
# value: "$(KIBANA_FLEET_PASSWORD)"

‎docs/local-k8s-testing.md

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Run/Test local build of agent on k8s cluster
2+
3+
## Prerequisites
4+
5+
- Install [skaffold](https://skaffold.dev/docs/install/)
6+
- Install [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
7+
- Install a local k8s distribution and create a cluster:
8+
- [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
9+
- [k3d](https://k3d.io/v5.5.1/#installation)
10+
- [minikube](https://minikube.sigs.k8s.io/docs/start/) (not tested)
11+
12+
## Quickstart (the hard way)
13+
14+
#### Standalone or managed mode
15+
There are 2 distinct profiles available (pretty self-explanatory):
16+
- `elastic-agent-standalone`
17+
- `elastic-agent-managed`
18+
19+
One of those profiles must always be specified in a skaffold command (using the `-p` or `--profile` option), for sake of brevity we are gonna list only examples with standalone profile
20+
21+
### Prepare environment variables
22+
In order to run agent on a local k8s cluster we need to set some environment variables in `deploy/skaffold/.env` (we can use `.env.example`). Those environment variables must point to a running elastic stack installation.
23+
24+
#### Standalone mode
25+
```shell
26+
# standalone elastic agent vars
27+
ES_HOST=https://<elasticsearch host>:443
28+
ES_USERNAME=elastic
29+
ES_PASSWORD=changeme
30+
```
31+
32+
#### Managed mode
33+
```shell
34+
# managed elastic agent vars
35+
FLEET_URL=https://<fleet host>:443
36+
FLEET_ENROLLMENT_TOKEN=<enrollment token>
37+
```
38+
39+
### Run agent
40+
41+
In order to just deploy agent on your local cluster, open a terminal and then execute
42+
43+
44+
```shell
45+
skaffold run -p elastic-agent-standalone
46+
```
47+
and the output should be something similar to this:
48+
```shell
49+
Generating tags...
50+
...
51+
Checking cache...
52+
- docker.elastic.co/beats/elastic-agent: Found Locally
53+
Starting test...
54+
Starting pre-render hooks...
55+
Completed pre-render hooks
56+
Tags used in deployment:
57+
...
58+
Starting deploy...
59+
...
60+
- serviceaccount/elastic-agent-standalone created
61+
- role.rbac.authorization.k8s.io/elastic-agent-standalone created
62+
- role.rbac.authorization.k8s.io/elastic-agent-standalone-kubeadm-config created
63+
- clusterrole.rbac.authorization.k8s.io/elastic-agent-standalone created
64+
- rolebinding.rbac.authorization.k8s.io/elastic-agent-standalone created
65+
- rolebinding.rbac.authorization.k8s.io/elastic-agent-standalone-kubeadm-config created
66+
- clusterrolebinding.rbac.authorization.k8s.io/elastic-agent-standalone created
67+
- configmap/agent-node-datastreams created
68+
- configmap/fleet-es-configmap-95bb9gfkkt created
69+
- daemonset.apps/elastic-agent-standalone created
70+
Waiting for deployments to stabilize...
71+
Deployments stabilized in 14.411071ms
72+
You can also run [skaffold run --tail] to get the logs
73+
```
74+
75+
Once you are done, remove the deployments with:
76+
```shell
77+
skaffold delete -p elastic-agent-standalone
78+
```
79+
80+
81+
### Debug agent
82+
The elastic agent can be debugged by connecting a debugger client to the debugger port (forwarded automatically on localhost).
83+
84+
If we run the command below (the `--tail=false` is there only to disable the streaming of logs on stdout):
85+
```shell
86+
skaffold debug -p elastic-agent-standalone --tail=false
87+
```
88+
we should have something similar to the output below
89+
90+
```shell
91+
Generating tags...
92+
...
93+
Checking cache...
94+
...
95+
Starting pre-render hooks...
96+
Completed pre-render hooks
97+
Tags used in deployment:
98+
...
99+
Starting deploy...
100+
Loading images into kind cluster nodes...
101+
...
102+
Images loaded in 35.802479ms
103+
... <a bunch of k8s resources created>
104+
Waiting for deployments to stabilize...
105+
Deployments stabilized in 7.654873ms
106+
Listing files to watch...
107+
- docker.elastic.co/beats/elastic-agent
108+
Press Ctrl+C to exit
109+
Not watching for changes...
110+
WARN[0003] unable to get owner from reference: {apps/v1 DaemonSet elastic-agent-standalone 0c579ec4-319a-4e85-99ec-c409260cb6ce 0xc000f950a0 0xc000f950a1} subtask=-1 task=DevLoop
111+
Port forwarding pod/elastic-agent-standalone-lgr9d in namespace kube-system, remote port 56268 -> http://127.0.0.1:56268
112+
```
113+
The last line of the output tells us where to connect our debugger ;)
114+
(it's always 56268 unless that port is busy, in which case skaffold will try to increment it until it finds one that can be bound).
115+
The debug session will continue until we hit `Ctrl+C` to stop the debug and start the cleanup
116+
117+
## Quickstart (the easy way)
118+
119+
Check [Google cloud code extension](https://cloud.google.com/code/docs) to do away with the terminal and to have Run/Debug configuration directly in your IDE (if supported)
120+

‎skaffold.yaml

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
apiVersion: skaffold/v4beta5
2+
kind: Config
3+
metadata:
4+
name: elastic-agent
5+
build:
6+
local:
7+
push: false
8+
useBuildkit: false
9+
useDockerCLI: true
10+
artifacts:
11+
- image: docker.elastic.co/beats/elastic-agent
12+
docker:
13+
dockerfile: Dockerfile.skaffold
14+
noCache: false
15+
pullParent: false
16+
squash: false
17+
profiles:
18+
- name: elastic-agent-standalone
19+
manifests:
20+
kustomize:
21+
paths: ["deploy/skaffold"]
22+
hooks:
23+
before:
24+
- host:
25+
os: ["linux", "macos"]
26+
command:
27+
- cp
28+
- deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
29+
- deploy/skaffold/elastic-agent-kubernetes.yaml
30+
- host:
31+
os: ["windows"]
32+
command:
33+
- copy
34+
- /Y
35+
- deploy\kubernetes\elastic-agent-standalone-kubernetes.yaml
36+
- deploy\skaffold\elastic-agent-kubernetes.yaml
37+
- name: elastic-agent-managed
38+
manifests:
39+
kustomize:
40+
paths: ["deploy/skaffold"]
41+
hooks:
42+
before:
43+
- host:
44+
os: ["linux", "macos"]
45+
command:
46+
- cp
47+
- deploy/kubernetes/elastic-agent-managed-kubernetes.yaml
48+
- deploy/skaffold/elastic-agent-kubernetes.yaml
49+
- host:
50+
os: ["windows"]
51+
command:
52+
- copy
53+
- /Y
54+
- deploy\kubernetes\elastic-agent-managed-kubernetes.yaml
55+
- deploy\skaffold\elastic-agent-kubernetes.yaml
56+
deploy:
57+
kubectl: {}

0 commit comments

Comments
 (0)