Check if non-snapshot version is released before using for testing (#4276)

Currently, we don't check whether a version without the `-SNAPSHOT` suffix
is actually released and available on our public CDN.

The version is present on the artifact API once the first BC is built,
however, this does not mean the version is published on the CDN.

This leads to failing upgrade attempts by the agent in our integration tests.

Author: rdner

pkg/testing/tools/artifacts_api.go

@@ -12,8 +12,10 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"runtime"
 	"sort"
 
+	"github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/version"
 )
 
@@ -26,6 +28,7 @@ const (
 	// artifactAPIV1SearchVersionPackage = "v1/search/%s/%s"
 
 	artifactElasticAgentProject = "elastic-agent-package"
+	artifactReleaseCDN          = "https://artifacts.elastic.co/downloads/beats/elastic-agent"
 )
 
 var (
@@ -119,6 +122,10 @@ func WithUrl(url string) ArtifactAPIClientOpt {
 	return func(aac *ArtifactAPIClient) { aac.url = url }
 }
 
+func WithCDNUrl(url string) ArtifactAPIClientOpt {
+	return func(aac *ArtifactAPIClient) { aac.cdnURL = url }
+}
+
 func WithHttpClient(client httpDoer) ArtifactAPIClientOpt {
 	return func(aac *ArtifactAPIClient) { aac.c = client }
 }
@@ -127,15 +134,17 @@ func WithHttpClient(client httpDoer) ArtifactAPIClientOpt {
 // More information about the API can be found at https://artifacts-api.elastic.co/v1
 // which will print a list of available operations
 type ArtifactAPIClient struct {
-	c   httpDoer
-	url string
+	c      httpDoer
+	url    string
+	cdnURL string
 }
 
 // NewArtifactAPIClient creates a new Artifact API client
 func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
 	c := &ArtifactAPIClient{
-		url: defaultArtifactAPIURL,
-		c:   new(http.Client),
+		url:    defaultArtifactAPIURL,
+		cdnURL: artifactReleaseCDN,
+		c:      new(http.Client),
 	}
 
 	for _, opt := range opts {
@@ -161,6 +170,68 @@ func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList
 	return checkResponseAndUnmarshal[VersionList](resp)
 }
 
+// RemoveUnreleasedVersions from the list
+// There is a period of time when a version is already marked as released
+// but not published on the CDN. This happens when we already have build candidates.
+// This function checks if a version marked as released actually has published artifacts.
+// If there are no published artifacts, the version is removed from the list.
+func (aac ArtifactAPIClient) RemoveUnreleasedVersions(ctx context.Context, vList *VersionList) error {
+	suffix, err := testing.GetPackageSuffix(runtime.GOOS, runtime.GOARCH)
+	if err != nil {
+		return fmt.Errorf("failed to generate the artifact suffix: %w", err)
+	}
+
+	results := make([]string, 0, len(vList.Versions))
+
+	for _, versionItem := range vList.Versions {
+		parsedVersion, err := version.ParseVersion(versionItem)
+		if err != nil {
+			return fmt.Errorf("failed to parse version %s: %w", versionItem, err)
+		}
+		// we check only release versions without `-SNAPSHOT`
+		if parsedVersion.Prerelease() != "" {
+			results = append(results, versionItem)
+			continue
+		}
+		url := fmt.Sprintf("%s/elastic-agent-%s-%s", aac.cdnURL, versionItem, suffix)
+		// using method `HEAD` to avoid downloading the file
+		req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
+		if err != nil {
+			return fmt.Errorf("failed to create an HTTP request to %q: %w", url, err)
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return fmt.Errorf("failed to request %q: %w", url, err)
+		}
+
+		// we don't read the response. However, we must drain when it's present,
+		// so the connection can be re-used later, see:
+		// https://cs.opensource.google/go/go/+/refs/tags/go1.22.0:src/net/http/response.go;l=62-64
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+
+		switch resp.StatusCode {
+		case http.StatusNotFound:
+			continue
+		case http.StatusOK:
+			results = append(results, versionItem)
+			continue
+		default:
+			return fmt.Errorf("unexpected status code from %s - %d", url, resp.StatusCode)
+		}
+	}
+
+	// nothing changed
+	if len(vList.Versions) == len(results) {
+		return nil
+	}
+
+	vList.Versions = results
+
+	return nil
+}
+
 // GetBuildsForVersion returns a list of builds for a specific version.
 // version should be one of the version strings returned by the GetVersions (expected format is semver
 // with optional prerelease but no build metadata, for example 8.9.0-SNAPSHOT)

pkg/testing/tools/artifacts_api_test.go

@@ -6,12 +6,15 @@ package tools
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -216,3 +219,74 @@ func TestDefaultArtifactAPIClient(t *testing.T) {
 	assert.NotEmpty(t, buildDetails.Build.Projects)
 	assert.Contains(t, buildDetails.Build.Projects, "elastic-agent")
 }
+
+func createVersionList(t *testing.T) *VersionList {
+	var list VersionList
+	err := json.Unmarshal([]byte(cannedVersions), &list)
+	require.NoError(t, err)
+	return &list
+}
+
+func TestRemoveUnreleasedVersions(t *testing.T) {
+	versionRegExp := regexp.MustCompile(`/elastic-agent-(\d+\.\d+\.\d+)-.+$`)
+	var unreleasedVersions map[string]struct{}
+	server := httptest.NewServer(http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		matches := versionRegExp.FindAllStringSubmatch(req.URL.Path, 2)
+		if len(matches) == 0 || len(matches[0]) < 2 {
+			resp.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		version := matches[0][1]
+		if _, unreleased := unreleasedVersions[version]; unreleased {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		resp.WriteHeader(http.StatusOK)
+	}))
+
+	client := NewArtifactAPIClient(WithCDNUrl(server.URL))
+
+	t.Run("removes unreleased versions", func(t *testing.T) {
+		unreleasedVersions = map[string]struct{}{
+			"8.6.1": {},
+			"8.8.0": {},
+		}
+
+		versionList := createVersionList(t)
+		err := client.RemoveUnreleasedVersions(context.Background(), versionList)
+		require.NoError(t, err)
+		exp := []string{
+			"7.17.9",
+			"7.17.10",
+			"8.6.0",
+			"8.6.2",
+			"8.7.0",
+			"8.7.1",
+			"8.8.1",
+			"8.9.0-SNAPSHOT",
+		}
+		require.Equal(t, exp, versionList.Versions)
+	})
+
+	t.Run("does not change the list if all released", func(t *testing.T) {
+		unreleasedVersions = map[string]struct{}{} // everything is released
+
+		versionList := createVersionList(t)
+		err := client.RemoveUnreleasedVersions(context.Background(), versionList)
+		require.NoError(t, err)
+		exp := []string{
+			"7.17.9",
+			"7.17.10",
+			"8.6.0",
+			"8.6.1",
+			"8.6.2",
+			"8.7.0",
+			"8.7.1",
+			"8.8.0",
+			"8.8.1",
+			"8.9.0-SNAPSHOT",
+		}
+		require.Equal(t, exp, versionList.Versions)
+	})
+}

testing/upgradetest/versions.go

@@ -44,6 +44,10 @@ func GetUpgradableVersions(ctx context.Context, upgradeToVersion string, current
 	if len(vList.Versions) == 0 {
 		return nil, errors.New("retrieved versions list from Artifact API is empty")
 	}
+	err = aac.RemoveUnreleasedVersions(ctx, vList)
+	if err != nil {
+		return nil, fmt.Errorf("failed to remove unreleased versions: %w", err)
+	}
 
 	return getUpgradableVersions(ctx, vList, upgradeToVersion, currentMajorVersions, previousMajorVersions)
 }