Use the public product versions API for creating the versions list (#4423)

Replacing the artifact API with the more stable source.

Author: rdner

.agent-versions.json

@@ -1,8 +1,10 @@
 {
   "testVersions": [
-    "8.14.0-SNAPSHOT",
     "8.13.0-SNAPSHOT",
+    "8.12.3-SNAPSHOT",
     "8.12.2",
+    "8.12.1",
+    "7.17.19-SNAPSHOT",
     "7.17.18"
   ]
 }

magefile.go

@@ -36,7 +36,9 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/multipass"
 	"github.com/elastic/elastic-agent/pkg/testing/ogc"
 	"github.com/elastic/elastic-agent/pkg/testing/runner"
-	"github.com/elastic/elastic-agent/pkg/testing/tools"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/git"
+	pv "github.com/elastic/elastic-agent/pkg/testing/tools/product_versions"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/snapshots"
 	"github.com/elastic/elastic-agent/pkg/version"
 	"github.com/elastic/elastic-agent/testing/upgradetest"
 	bversion "github.com/elastic/elastic-agent/version"
@@ -1576,17 +1578,37 @@ func (Integration) Single(ctx context.Context, testName string) error {
 // UpdateVersions runs an update on the `.agent-versions.json` fetching
 // the latest version list from the artifact API.
 func (Integration) UpdateVersions(ctx context.Context) error {
-	// test 2 current 8.x version, 1 previous 7.x version and 1 recent snapshot
+	maxSnapshots := 3
+
+	branches, err := git.GetReleaseBranches(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list release branches: %w", err)
+	}
+
+	// -1 because we manually add 7.17 below
+	if len(branches) > maxSnapshots-1 {
+		branches = branches[:maxSnapshots-1]
+	}
+
+	// it's not a part of this repository, cannot be retrieved with `GetReleaseBranches`
+	branches = append(branches, "7.17")
+
+	// uncomment if want to have the current version snapshot on the list as well
+	// branches = append([]string{"master"}, branches...)
+
 	reqs := upgradetest.VersionRequirements{
 		UpgradeToVersion: bversion.Agent,
 		CurrentMajors:    2,
 		PreviousMinors:   1,
 		PreviousMajors:   1,
-		RecentSnapshots:  1,
+		SnapshotBranches: branches,
 	}
+	b, _ := json.MarshalIndent(reqs, "", "  ")
+	fmt.Printf("Current version requirements: \n%s\n", b)
 
-	aac := tools.NewArtifactAPIClient(tools.WithLogFunc(log.Default().Printf))
-	versions, err := upgradetest.FetchUpgradableVersions(ctx, aac, reqs)
+	pvc := pv.NewProductVersionsClient()
+	sc := snapshots.NewSnapshotsClient()
+	versions, err := upgradetest.FetchUpgradableVersions(ctx, pvc, sc, reqs)
 	if err != nil {
 		return fmt.Errorf("failed to fetch upgradable versions: %w", err)
 	}

pkg/testing/tools/artifacts_api.go

@@ -9,34 +9,24 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
-	"sort"
 	"time"
-
-	"github.com/elastic/elastic-agent/pkg/version"
 )
 
 const (
 	defaultArtifactAPIURL = "https://artifacts-api.elastic.co/"
 
-	artifactsAPIV1VersionsEndpoint      = "v1/versions/"
 	artifactsAPIV1VersionBuildsEndpoint = "v1/versions/%s/builds/"
 	artifactAPIV1BuildDetailsEndpoint   = "v1/versions/%s/builds/%s"
-	// artifactAPIV1SearchVersionPackage = "v1/search/%s/%s"
 
 	artifactElasticAgentProject      = "elastic-agent-package"
 	maxAttemptsForArtifactsAPICall   = 6
 	retryIntervalForArtifactsAPICall = 5 * time.Second
 )
 
 var (
-	ErrLatestVersionNil        = errors.New("latest version is nil")
-	ErrSnapshotVersionsEmpty   = errors.New("snapshot list is nil")
-	ErrInvalidVersionRetrieved = errors.New("invalid version retrieved from artifact API")
-	ErrBuildNotFound           = errors.New("there are no build that satisfy given conditions")
-
+	ErrBuildNotFound     = errors.New("there are no builds that satisfy given conditions")
 	ErrBadHTTPStatusCode = errors.New("bad http status code")
 )
 
@@ -45,12 +35,6 @@ type Manifests struct {
 	SecondsSinceLastUpdate int    `json:"seconds-since-last-update"`
 }
 
-type VersionList struct {
-	Versions  []string  `json:"versions"`
-	Aliases   []string  `json:"aliases"`
-	Manifests Manifests `json:"manifests"`
-}
-
 type VersionBuilds struct {
 	Builds    []string  `json:"builds"`
 	Manifests Manifests `json:"manifests"`
@@ -107,11 +91,6 @@ type BuildDetails struct {
 	Manifests Manifests `json:"manifests"`
 }
 
-type SearchPackageResult struct {
-	Packages  map[string]Package `json:"packages"`
-	Manifests Manifests          `json:"manifests"`
-}
-
 type httpDoer interface {
 	Do(req *http.Request) (*http.Response, error)
 }
@@ -156,26 +135,10 @@ func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
 	return c
 }
 
-// GetVersions returns a list of versions as server by the Artifact API along with some aliases and manifest information
-func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList, err error) {
-	joinedURL, err := aac.composeURL(artifactsAPIV1VersionsEndpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
-	if err != nil {
-		return nil, fmt.Errorf("getting versions: %w", err)
-	}
-
-	defer resp.Body.Close()
-	return checkResponseAndUnmarshal[VersionList](resp)
-}
-
 // GetBuildsForVersion returns a list of builds for a specific version.
 // version should be one of the version strings returned by the GetVersions (expected format is semver
 // with optional prerelease but no build metadata, for example 8.9.0-SNAPSHOT)
-func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version string) (builds *VersionBuilds, err error) {
+func (aac ArtifactAPIClient) getBuildsForVersion(ctx context.Context, version string) (builds *VersionBuilds, err error) {
 	joinedURL, err := aac.composeURL(fmt.Sprintf(artifactsAPIV1VersionBuildsEndpoint, version))
 	if err != nil {
 		return nil, err
@@ -196,15 +159,15 @@ func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version st
 // Setting `offset` to 0 includes all builds, 1 skips the latest, and so forth.
 // If there are no builds matching these conditions, returns `ErrBuildNotFound`.
 func (aac ArtifactAPIClient) FindBuild(ctx context.Context, version, excludeHash string, offset int) (buildDetails *BuildDetails, err error) {
-	resp, err := aac.GetBuildsForVersion(ctx, version)
+	resp, err := aac.getBuildsForVersion(ctx, version)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get a list of builds: %w", err)
 	}
 	if len(resp.Builds) < offset+1 {
 		return nil, ErrBuildNotFound
 	}
 	for _, buildID := range resp.Builds[offset:] {
-		details, err := aac.GetBuildDetails(ctx, version, buildID)
+		details, err := aac.getBuildDetails(ctx, version, buildID)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get build information for %q: %w", buildID, err)
 		}
@@ -219,7 +182,7 @@ func (aac ArtifactAPIClient) FindBuild(ctx context.Context, version, excludeHash
 // GetBuildDetails returns the list of project and artifacts related to a specific build.
 // Version parameter format follows semver (without build metadata) and buildID format is <major>.<minor>.<patch>-<buildhash> as returned by
 // GetBuildsForVersion()
-func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string, buildID string) (buildDetails *BuildDetails, err error) {
+func (aac ArtifactAPIClient) getBuildDetails(ctx context.Context, version string, buildID string) (buildDetails *BuildDetails, err error) {
 	joinedURL, err := aac.composeURL(fmt.Sprintf(artifactAPIV1BuildDetailsEndpoint, version, buildID))
 	if err != nil {
 		return nil, err
@@ -291,59 +254,12 @@ func checkResponseAndUnmarshal[T any](resp *http.Response) (*T, error) {
 		return nil, fmt.Errorf("%d: %w", resp.StatusCode, ErrBadHTTPStatusCode)
 	}
 
-	respBytes, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("reading response body: %w", err)
-	}
+	d := json.NewDecoder(resp.Body)
 	result := new(T)
-	err = json.Unmarshal(respBytes, result)
-
+	err := d.Decode(&result)
 	if err != nil {
-		return nil, fmt.Errorf("unmarshaling: %w", err)
+		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
 
 	return result, nil
 }
-
-func (aac ArtifactAPIClient) GetLatestSnapshotVersion(ctx context.Context) (*version.ParsedSemVer, error) {
-	vList, err := aac.GetVersions(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	if vList == nil {
-		return nil, ErrSnapshotVersionsEmpty
-	}
-
-	sortedParsedVersions := make(version.SortableParsedVersions, 0, len(vList.Versions))
-	for _, v := range vList.Versions {
-		pv, err := version.ParseVersion(v)
-		if err != nil {
-			aac.logFunc("invalid version retrieved from artifact API: %q", v)
-			return nil, ErrInvalidVersionRetrieved
-		}
-		sortedParsedVersions = append(sortedParsedVersions, pv)
-	}
-
-	if len(sortedParsedVersions) == 0 {
-		return nil, ErrSnapshotVersionsEmpty
-	}
-
-	// normally the output of the versions returned by artifact API is already
-	// sorted in ascending order.If we want to sort in descending order we need
-	// to pass a sort.Reverse to sort.Sort.
-	sort.Sort(sort.Reverse(sortedParsedVersions))
-
-	var latestSnapshotVersion *version.ParsedSemVer
-	// fetch the latest SNAPSHOT build
-	for _, pv := range sortedParsedVersions {
-		if pv.IsSnapshot() {
-			latestSnapshotVersion = pv
-			break
-		}
-	}
-	if latestSnapshotVersion == nil {
-		return nil, ErrLatestVersionNil
-	}
-	return latestSnapshotVersion, nil
-}