Skip to content

Commit 6ea9ea0

Browse files
committed
Use the public product versions API for creating the versions list
Replacing the artifact API with the more stable source.
1 parent a7034ef commit 6ea9ea0

13 files changed

+655
-69
lines changed

.agent-versions.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"testVersions": [
3-
"8.14.0-SNAPSHOT",
43
"8.13.0-SNAPSHOT",
4+
"8.12.3-SNAPSHOT",
55
"8.12.2",
6+
"8.12.1",
67
"7.17.18"
78
]
89
}

magefile.go

+18-4
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@ import (
3636
"github.com/elastic/elastic-agent/pkg/testing/multipass"
3737
"github.com/elastic/elastic-agent/pkg/testing/ogc"
3838
"github.com/elastic/elastic-agent/pkg/testing/runner"
39-
"github.com/elastic/elastic-agent/pkg/testing/tools"
39+
"github.com/elastic/elastic-agent/pkg/testing/tools/git"
40+
pv "github.com/elastic/elastic-agent/pkg/testing/tools/product_versions"
41+
"github.com/elastic/elastic-agent/pkg/testing/tools/snapshots"
4042
"github.com/elastic/elastic-agent/pkg/version"
4143
"github.com/elastic/elastic-agent/testing/upgradetest"
4244
bversion "github.com/elastic/elastic-agent/version"
@@ -1576,17 +1578,29 @@ func (Integration) Single(ctx context.Context, testName string) error {
15761578
// UpdateVersions runs an update on the `.agent-versions.json` fetching
15771579
// the latest version list from the artifact API.
15781580
func (Integration) UpdateVersions(ctx context.Context) error {
1581+
branches, err := git.GetReleaseBranches(ctx)
1582+
if err != nil {
1583+
return fmt.Errorf("failed to list release branches: %w", err)
1584+
}
1585+
1586+
// uncomment if want to have the current version snapshot on the list as well
1587+
// branches = append([]string{"master"}, branches...)
1588+
15791589
// test 2 current 8.x version, 1 previous 7.x version and 1 recent snapshot
15801590
reqs := upgradetest.VersionRequirements{
15811591
UpgradeToVersion: bversion.Agent,
15821592
CurrentMajors: 2,
15831593
PreviousMinors: 1,
15841594
PreviousMajors: 1,
1585-
RecentSnapshots: 1,
1595+
RecentSnapshots: 2,
1596+
ReleaseBranches: branches,
15861597
}
1598+
b, _ := json.MarshalIndent(reqs, "", " ")
1599+
fmt.Printf("Current version requirements: \n%s\n", b)
15871600

1588-
aac := tools.NewArtifactAPIClient(tools.WithLogFunc(log.Default().Printf))
1589-
versions, err := upgradetest.FetchUpgradableVersions(ctx, aac, reqs)
1601+
pvc := pv.NewProductVersionsClient()
1602+
sc := snapshots.NewSnapshotsClient()
1603+
versions, err := upgradetest.FetchUpgradableVersions(ctx, pvc, sc, reqs)
15901604
if err != nil {
15911605
return fmt.Errorf("failed to fetch upgradable versions: %w", err)
15921606
}

pkg/testing/tools/git/git.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package git
6+
7+
import (
8+
"bufio"
9+
"context"
10+
"fmt"
11+
"os/exec"
12+
"regexp"
13+
)
14+
15+
var (
16+
releaseBranchRegexp = regexp.MustCompile(`.*(\d+\.\d+)$`)
17+
)
18+
19+
// GetReleaseBranches returns a list of release branches in the current repository.
20+
// e.g. 8.12, 8.13, etc.
21+
func GetReleaseBranches(ctx context.Context) ([]string, error) {
22+
var seen = map[string]struct{}{}
23+
branchList := []string{}
24+
25+
c := exec.CommandContext(ctx, "git", "branch", "-r", "--list", "*/[0-9]*.*[0-9]")
26+
27+
r, err := c.StdoutPipe()
28+
if err != nil {
29+
return nil, fmt.Errorf("failed to create the stdout pipe: %w", err)
30+
}
31+
defer r.Close()
32+
33+
err = c.Start()
34+
if err != nil {
35+
return nil, fmt.Errorf("failed to start git command: %w", err)
36+
}
37+
38+
scanner := bufio.NewScanner(r)
39+
for scanner.Scan() {
40+
branch := scanner.Text()
41+
if !releaseBranchRegexp.MatchString(branch) {
42+
continue
43+
}
44+
45+
matches := releaseBranchRegexp.FindStringSubmatch(branch)
46+
if len(matches) != 2 {
47+
continue
48+
}
49+
branch = matches[1]
50+
_, exists := seen[branch]
51+
if exists {
52+
continue
53+
}
54+
seen[branch] = struct{}{}
55+
// appending to the list right away instead of
56+
// collecting from the map later preserves the order
57+
branchList = append(branchList, branch)
58+
}
59+
if scanner.Err() != nil {
60+
return nil, fmt.Errorf("failed to scan the output: %w", err)
61+
}
62+
63+
err = c.Wait()
64+
if err != nil {
65+
return nil, fmt.Errorf("failed to wait for the git command to finish: %w", err)
66+
}
67+
68+
return branchList, nil
69+
}

pkg/testing/tools/git/git_test.go

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package git
6+
7+
import (
8+
"context"
9+
"testing"
10+
"time"
11+
12+
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
func TestGetReleaseBranches(t *testing.T) {
17+
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
18+
defer cancel()
19+
branches, err := GetReleaseBranches(ctx)
20+
require.NoError(t, err)
21+
t.Log(branches)
22+
assert.NotEmpty(t, branches)
23+
for _, b := range branches {
24+
assert.Regexp(t, releaseBranchRegexp, b)
25+
}
26+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package product_versions
6+
7+
import (
8+
"context"
9+
"encoding/json"
10+
"fmt"
11+
"net/http"
12+
13+
"github.com/elastic/elastic-agent/pkg/version"
14+
)
15+
16+
const (
17+
// Every product on the version list has a unique product ID
18+
// This product ID belongs to Elastic Agent excluding alpha/beta/RC versions.
19+
elasticAgentProductID = "bltce270507523f4c56"
20+
productVersionsAPIURL = "https://www.elastic.co/api/product_versions"
21+
)
22+
23+
type item struct {
24+
// Version contains the actual semantic version, e.g. `8.12.1`
25+
Version string `json:"version_number"`
26+
// Product contains a list of product IDs, for the agent it should
27+
// be a single item that equals `elasticAgentProductID`
28+
Product []string `json:"product"`
29+
}
30+
31+
type httpDoer interface {
32+
Do(req *http.Request) (*http.Response, error)
33+
}
34+
35+
type ProductVersionsClientOpt func(pvc *ProductVersionsClient)
36+
37+
func WithUrl(url string) ProductVersionsClientOpt {
38+
return func(pvc *ProductVersionsClient) { pvc.url = url }
39+
}
40+
41+
func WithHttpClient(client httpDoer) ProductVersionsClientOpt {
42+
return func(pvc *ProductVersionsClient) { pvc.c = client }
43+
}
44+
45+
type ProductVersionsClient struct {
46+
c httpDoer
47+
url string
48+
}
49+
50+
// NewProductVersionsClient creates a new client applying all the given options.
51+
// If not set by the options, the new client will use the default HTTP client and
52+
// the default URL from `productVersionsAPIURL`.
53+
//
54+
// All the timeout/retry/backoff behavior must be implemented by the `httpDoer` interface
55+
// and set by the `WithClient` option.
56+
func NewProductVersionsClient(opts ...ProductVersionsClientOpt) *ProductVersionsClient {
57+
c := &ProductVersionsClient{
58+
url: productVersionsAPIURL,
59+
c: http.DefaultClient,
60+
}
61+
62+
for _, opt := range opts {
63+
opt(c)
64+
}
65+
66+
return c
67+
}
68+
69+
// FetchAgentVersions returns a sortable list of parsed semantic versions for Elastic Agent.
70+
// This list contains only publicly available versions/releases ordered by their creation date.
71+
func (pvc *ProductVersionsClient) FetchAgentVersions(ctx context.Context) (version.SortableParsedVersions, error) {
72+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pvc.url, nil)
73+
if err != nil {
74+
err = fmt.Errorf("failed to create request: %w", err)
75+
return nil, err
76+
}
77+
78+
resp, err := pvc.c.Do(req)
79+
if err != nil {
80+
return nil, fmt.Errorf("failed to do request: %w", err)
81+
}
82+
defer resp.Body.Close()
83+
if resp.StatusCode != http.StatusOK {
84+
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
85+
}
86+
87+
// The body is large (> 15MB), so streaming decoder is used
88+
d := json.NewDecoder(resp.Body)
89+
90+
var versions [][]item
91+
err = d.Decode(&versions)
92+
if err != nil {
93+
return nil, fmt.Errorf("failed to decode JSON: %w", err)
94+
}
95+
if len(versions) == 0 {
96+
return []*version.ParsedSemVer{}, nil
97+
}
98+
99+
var versionList version.SortableParsedVersions
100+
// there are 2 array levels
101+
for _, i := range versions {
102+
for _, v := range i {
103+
if len(v.Product) != 1 {
104+
continue
105+
}
106+
if v.Product[0] != elasticAgentProductID {
107+
continue
108+
}
109+
parsed, err := version.ParseVersion(v.Version)
110+
if err != nil {
111+
return nil, fmt.Errorf("failed to parse %s: %w", v.Version, err)
112+
}
113+
versionList = append(versionList, parsed)
114+
}
115+
}
116+
117+
return versionList, nil
118+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package product_versions
6+
7+
import (
8+
"bufio"
9+
"context"
10+
"net/http"
11+
"net/http/httptest"
12+
"os"
13+
"testing"
14+
"time"
15+
16+
"github.com/stretchr/testify/assert"
17+
"github.com/stretchr/testify/require"
18+
19+
"github.com/elastic/elastic-agent/pkg/version"
20+
)
21+
22+
func TestFetchAgentVersions(t *testing.T) {
23+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
24+
defer cancel()
25+
26+
versionResponse, err := os.ReadFile("./testdata/versions.json")
27+
require.NoError(t, err)
28+
29+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
30+
written, err := w.Write(versionResponse)
31+
assert.NoError(t, err)
32+
assert.Equal(t, len(versionResponse), written)
33+
}))
34+
pvc := NewProductVersionsClient(WithUrl(server.URL))
35+
36+
versions, err := pvc.FetchAgentVersions(ctx)
37+
require.NoError(t, err)
38+
assert.NotEmpty(t, versions)
39+
expectedVersions := readExpectedVersions(t)
40+
assert.NotEmpty(t, expectedVersions)
41+
assert.Equal(t, expectedVersions, versions)
42+
}
43+
44+
// readExpectedVersions returns a prepared list of versions that should match the `FetchAgentVersions` output
45+
func readExpectedVersions(t *testing.T) version.SortableParsedVersions {
46+
var expectedVersions version.SortableParsedVersions
47+
expectedVersionsFile, err := os.Open("./testdata/expected-versions.txt")
48+
require.NoError(t, err)
49+
defer expectedVersionsFile.Close()
50+
51+
scanner := bufio.NewScanner(expectedVersionsFile)
52+
for scanner.Scan() {
53+
v, err := version.ParseVersion(scanner.Text())
54+
require.NoError(t, err)
55+
expectedVersions = append(expectedVersions, v)
56+
}
57+
require.NoError(t, scanner.Err())
58+
59+
return expectedVersions
60+
}

0 commit comments

Comments
 (0)