Skip to content

Commit 72653ac

Browse files
pazonepazone
authored
[CI]Buildkite k8s tests migration (#3680)
* Buildkite k8s tests * Buildkite k8s tests * Added kind setup script * Added kind setup script * Added kind setup script * Added kind setup script * Branch configuration for k8s tests * transformed to matrix build * transformed to matrix build * transformed to matrix build * transformed to matrix build * transformed to matrix build * transformed to matrix build * transformed to matrix build * Removed k8s tests from Jenkinsfile * Run k8s tests on every PR * Fix review comments * install kind: moved path update to the top * Moved PATH declaration to the top leve script * Removed junit-annotate from k8s tests * Moved k8s tests to upper level in the pipeline
1 parent b0b8e85 commit 72653ac

File tree

6 files changed

+151
-58
lines changed

6 files changed

+151
-58
lines changed

‎.buildkite/pipeline.yml

+25
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,31 @@ steps:
122122
- unit-tests-macos-13-arm
123123
allow_dependency_failure: true
124124

125+
- group: "K8s tests"
126+
key: "k8s-tests"
127+
steps:
128+
- label: "K8s tests: {{matrix.k8s_version}}"
129+
env:
130+
K8S_VERSION: "v{{matrix.k8s_version}}"
131+
KIND_VERSION: "v0.20.0"
132+
command: ".buildkite/scripts/steps/k8s-tests.sh"
133+
artifact_paths:
134+
- "build/TEST-**"
135+
- "build/diagnostics/*"
136+
- "coverage.out"
137+
agents:
138+
provider: "gcp"
139+
image: "family/core-ubuntu-2204"
140+
matrix:
141+
setup:
142+
k8s_version:
143+
- "1.28.0"
144+
- "1.27.3"
145+
- "1.26.6"
146+
retry:
147+
manual:
148+
allowed: true
149+
125150
- label: ":sonarqube: Continuous Code Inspection"
126151
env:
127152
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"

‎.buildkite/scripts/install-kind.sh

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env bash
2+
set -exuo pipefail
3+
4+
echo "--- Install Kind"
5+
6+
MSG="environment variable missing."
7+
DEFAULT_HOME="/usr/local"
8+
KIND_VERSION=${KIND_VERSION:?$MSG}
9+
HOME=${HOME:?$DEFAULT_HOME}
10+
KIND_CMD="${HOME}/bin/kind"
11+
12+
if command -v kind
13+
then
14+
set +e
15+
echo "Found Kind. Checking version.."
16+
FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
17+
if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
18+
then
19+
echo "Versions match. No need to install Kind. Exiting."
20+
exit 0
21+
fi
22+
set -e
23+
fi
24+
25+
echo "Installing Kind"
26+
27+
OS=$(uname -s| tr '[:upper:]' '[:lower:]')
28+
ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
29+
if [ "${ARCH}" == "aarch64" ] ; then
30+
ARCH_SUFFIX=arm64
31+
else
32+
ARCH_SUFFIX=amd64
33+
fi
34+
35+
mkdir -p "${HOME}/bin"
36+
37+
if curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
38+
chmod +x "${KIND_CMD}"
39+
else
40+
echo "Something bad with the download, let's delete the corrupted binary"
41+
if [ -e "${KIND_CMD}" ] ; then
42+
rm "${KIND_CMD}"
43+
fi
44+
exit 1
45+
fi

‎.buildkite/scripts/install-kubectl.sh

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
echo "--- Install kubectl"
5+
6+
MSG="parameter missing."
7+
DEFAULT_HOME="/usr/local"
8+
K8S_VERSION=${K8S_VERSION:?$MSG}
9+
HOME=${HOME:?$DEFAULT_HOME}
10+
KUBECTL_CMD="${HOME}/bin/kubectl"
11+
12+
if command -v kubectl
13+
then
14+
set +e
15+
echo "Found kubectl. Checking version.."
16+
FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
17+
if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
18+
then
19+
echo "Versions match. No need to install kubectl. Exiting."
20+
exit 0
21+
fi
22+
set -e
23+
fi
24+
25+
echo "Installing kubectl"
26+
27+
mkdir -p "${HOME}/bin"
28+
29+
OS=$(uname -s| tr '[:upper:]' '[:lower:]')
30+
ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
31+
if [ "${ARCH}" == "aarch64" ] ; then
32+
ARCH_SUFFIX=arm64
33+
else
34+
ARCH_SUFFIX=amd64
35+
fi
36+
37+
if curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then
38+
chmod +x "${KUBECTL_CMD}"
39+
else
40+
echo "Something bad with the download, let's delete the corrupted binary"
41+
if [ -e "${KUBECTL_CMD}" ] ; then
42+
rm "${KUBECTL_CMD}"
43+
fi
44+
exit 1
45+
fi

‎.buildkite/scripts/steps/k8s-tests.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
export PATH=$HOME/bin:${PATH}
5+
source .buildkite/scripts/install-kubectl.sh
6+
source .buildkite/scripts/install-kind.sh
7+
8+
kind create cluster --image "kindest/node:${K8S_VERSION}" --config - <<EOF
9+
kind: Cluster
10+
apiVersion: kind.x-k8s.io/v1alpha4
11+
nodes:
12+
- role: control-plane
13+
kubeadmConfigPatches:
14+
- |
15+
kind: ClusterConfiguration
16+
scheduler:
17+
extraArgs:
18+
bind-address: "0.0.0.0"
19+
port: "10251"
20+
secure-port: "10259"
21+
controllerManager:
22+
extraArgs:
23+
bind-address: "0.0.0.0"
24+
port: "10252"
25+
secure-port: "10257"
26+
EOF
27+
kubectl cluster-info
28+
29+
30+
make -C deploy/kubernetes test

‎.buildkite/scripts/steps/unit-tests.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#!/usr/bin/env bash
2-
set -euxo pipefail
2+
set -uo pipefail
33

44
source .buildkite/scripts/common.sh
55

66
echo "--- Unit tests"
77
RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
8+
TESTS_EXIT_STATUS=$?
89
# Copy coverage file to build directory so it can be downloaded as an artifact
9-
cp build/TEST-go-unit.cov coverage.out
10+
cp build/TEST-go-unit.cov coverage.out
11+
exit $TESTS_EXIT_STATUS

‎.ci/Jenkinsfile

+2-56
Original file line numberDiff line numberDiff line change
@@ -95,26 +95,7 @@ pipeline {
9595
values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
9696
}
9797
}
98-
stages {
99-
stage('K8s') {
100-
when {
101-
beforeAgent true
102-
// Always when running builds on branches/tags
103-
// Enable if k8s related changes.
104-
allOf {
105-
expression { return env.PLATFORM == 'ubuntu-22 && immutable' }
106-
anyOf {
107-
not { changeRequest() } // If no PR
108-
expression { return env.K8S_CHANGES == "true" }
109-
}
110-
}
111-
}
112-
steps {
113-
deleteDir()
114-
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
115-
runK8s(k8sVersion: 'v1.28.0', kindVersion: 'v0.20.0', context: "K8s-${PLATFORM}")
116-
}
117-
}
98+
stages {
11899
stage('Package') {
119100
when {
120101
beforeAgent true
@@ -167,42 +148,7 @@ pipeline {
167148
}
168149
}
169150
}
170-
}
171-
stage('Full K8s') {
172-
when {
173-
// Always when running builds on branches/tags
174-
// Enable if k8s related changes.
175-
anyOf {
176-
not { changeRequest() } // If no PR
177-
expression { return env.K8S_CHANGES == "true" } // If k8s changes
178-
}
179-
}
180-
failFast false
181-
matrix {
182-
agent {label 'ubuntu-22 && immutable'}
183-
options { skipDefaultCheckout() }
184-
axes {
185-
axis {
186-
name 'K8S_VERSION'
187-
values "v1.28.0","v1.27.3","v1.26.6"
188-
}
189-
}
190-
stages {
191-
stage('K8s') {
192-
steps {
193-
deleteDir()
194-
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
195-
runK8s(k8sVersion: K8S_VERSION, kindVersion: 'v0.20.0', context: "K8s-${K8S_VERSION}")
196-
}
197-
post {
198-
always {
199-
junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
200-
}
201-
}
202-
}
203-
}
204-
}
205-
}
151+
}
206152
stage('Sync K8s') { //This stage opens a PR to kibana Repository in order to sync k8s manifests
207153
when {
208154
// Only on main branch

0 commit comments

Comments
 (0)