Skip to content

Commit 29c1d88

Browse files
pchiladliappis
pchila
and
dliappis
authored
Allow use of version qualifier for staging artifacts (#4328)
This commits adds support for an optional $VERSION_QUALIFIER env var/build option in the packaging pipeline to allow building prerelease artifacts from staging. Closes: - #4325 - elastic/ingest-dev#4859 --------- Co-authored-by: Dimitrios Liappis <dimitrios.liappis@gmail.com>
1 parent 2f2f429 commit 29c1d88

File tree

5 files changed

+54
-38
lines changed

5 files changed

+54
-38
lines changed

‎.buildkite/hooks/pre-command

+6-3
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
3838
fi
3939

4040
if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
41-
check_if_file_exist_in_repo "infra" "${BUILDKITE_BRANCH}" #TODO should be changed to "main" for rollback...
41+
# TODO remove and replace _branch with BUILDKITE_BRANCH after PR tests
42+
_branch="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
43+
44+
check_if_file_exist_in_repo "infra" "${_branch}" #TODO should be changed to "main" for rollback...
4245
fi
4346

4447
if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
@@ -70,7 +73,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "
7073
fi
7174

7275
if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
73-
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
76+
if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
7477
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
7578
export JOB_GCS_BUCKET
7679
fi
@@ -89,7 +92,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
8992
fi
9093

9194
if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
92-
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
95+
if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* ]]; then
9396
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
9497
export JOB_GCS_BUCKET
9598
fi

‎.buildkite/hooks/pre-exit

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "
1616
fi
1717

1818
if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
19-
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
19+
if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
2020
unset GOOGLE_APPLICATION_CREDENTIALS
2121
unset VAULT_ROLE_ID_SECRET
2222
unset VAULT_ADDR_SECRET

‎.buildkite/pipeline.package.mbp.yml

+44-31
Original file line numberDiff line numberDiff line change
@@ -8,55 +8,68 @@ env:
88
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
99

1010
steps:
11-
- group: "Package and Publish"
12-
key: "package-publish"
13-
steps:
14-
- label: "Package Ubuntu-20 x86_64"
15-
key: "package-x86-64"
16-
command: ".buildkite/scripts/package.sh {{matrix.type}}"
17-
agents:
18-
provider: "gcp"
19-
image: "${IMAGE_UBUNTU_X86_64}"
20-
machineType: "c2-standard-16"
21-
matrix:
22-
setup:
23-
type:
24-
- "snapshot"
25-
- "staging"
11+
- label: "Package x86_64 snapshot"
12+
# skip building + packaging snapshot for pre-releases (flagged by a non-empty VERSION_QUALIFIER env var/BK param)
13+
# as prereleases are only intended to be used with staging; details in https://github.com/elastic/ingest-dev/issues/4855
14+
if: "build.env('VERSION_QUALIFIER') == null"
15+
key: "package-x86-64-snapshot"
16+
command: ".buildkite/scripts/package.sh snapshot"
17+
agents:
18+
provider: "gcp"
19+
image: "${IMAGE_UBUNTU_X86_64}"
20+
machineType: "c2-standard-16"
21+
22+
- label: "Package x86_64 staging"
23+
key: "package-x86-64-staging"
24+
command: ".buildkite/scripts/package.sh snapshot"
25+
agents:
26+
provider: "gcp"
27+
image: "${IMAGE_UBUNTU_X86_64}"
28+
machineType: "c2-standard-16"
2629

27-
- label: "Package Ubuntu-20 aarch64"
28-
key: "package-arm"
29-
command: ".buildkite/scripts/package.sh {{matrix.type}}"
30-
agents:
31-
provider: "aws"
32-
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
33-
instanceType: "t4g.2xlarge"
34-
matrix:
35-
setup:
36-
type:
37-
- "snapshot"
38-
- "staging"
30+
- label: "Package aarch64 snapshot"
31+
if: "build.env('VERSION_QUALIFIER') == null"
32+
key: "package-arm-snapshot"
33+
command: ".buildkite/scripts/package.sh snapshot"
34+
agents:
35+
provider: "aws"
36+
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
37+
instanceType: "t4g.2xlarge"
38+
39+
- label: "Package aarch64 staging"
40+
key: "package-arm-staging"
41+
command: ".buildkite/scripts/package.sh staging"
42+
agents:
43+
provider: "aws"
44+
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
45+
instanceType: "t4g.2xlarge"
3946

4047
- label: "DRA snapshot"
48+
if: "${FILE_EXISTS_IN_REPO} && build.env('VERSION_QUALIFIER') == null"
4149
key: "dra-snapshot"
4250
command: ".buildkite/scripts/dra_release.sh snapshot"
4351
agents:
4452
provider: "gcp"
4553
image: "${IMAGE_UBUNTU_X86_64}"
4654
machineType: "c2-standard-16"
47-
if: ${FILE_EXISTS_IN_REPO}
4855
depends_on:
49-
- step: "package-publish"
56+
- step: "package-x86-64-snapshot"
57+
allow_failure: false
58+
- step: "package-arm-snapshot"
5059
allow_failure: false
5160

5261
- label: "DRA release staging"
62+
# we don't usually build staging from the main branch, but we exceptionally allow it for prereleases
63+
# details in https://github.com/elastic/ingest-dev/issues/4855
64+
if: "${FILE_EXISTS_IN_REPO} == true && (build.env('BUILDKITE_BRANCH') != 'main' || build.env('VERSION_QUALIFIER') != null)"
5365
key: "dra-staging"
5466
command: ".buildkite/scripts/dra_release.sh staging"
5567
agents:
5668
provider: "gcp"
5769
image: "${IMAGE_UBUNTU_X86_64}"
5870
machineType: "c2-standard-16"
59-
if: "${FILE_EXISTS_IN_REPO} == true && build.env('BUILDKITE_BRANCH') != 'main'"
6071
depends_on:
61-
- step: "dra-snapshot"
72+
- step: "package-x86-64-staging"
73+
allow_failure: false
74+
- step: "package-arm-staging"
6275
allow_failure: false

‎.buildkite/scripts/dra_release.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ BASE_DIR="${WORKSPACE}/${FOLDER_PATH}"
99
DRA_OUTPUT="release-manager.out"
1010
export PROJECT="fleet-server"
1111
export TYPE=${1}
12-
export BRANCH="${BUILDKITE_BRANCH}"
12+
# DRA_BRANCH can be used for manually testing packaging with PRs
13+
# e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
14+
export BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
1315
export VERSION="$(make get-version)"
1416

1517
if [[ "${VERSION}" == *"-SNAPSHOT"* || "${VERSION}" == "" ]]; then

‎.buildkite/scripts/package.sh

-2
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,8 @@ set -euo pipefail
44

55
source .buildkite/scripts/common.sh
66

7-
VERSION=$(awk '/const DefaultVersion/{print $NF}' version/version.go | tr -d '"')
87
PLATFORM_TYPE=$(uname -m)
98
TYPE="$1"
10-
INFRA_REPO="https://github.com/repos/elastic/infra/contents"
119

1210
if [[ ${BUILDKITE_BRANCH} == "main" && ${TYPE} == "staging" ]]; then
1311
echo "INFO: staging artifacts for the main branch are not required."

0 commit comments

Comments
 (0)