Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use the public product versions API for creating the versions list #4423

Merged
merged 5 commits into from
Mar 21, 2024
Merged

Use the public product versions API for creating the versions list #4423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b17b3ac
Use the public product versions API for creating the versions list
rdner Mar 18, 2024
d6b0484
Change snapshot requirements
rdner Mar 19, 2024
18c5c54
Remove redundant helper function
rdner Mar 19, 2024
c794505
Address PR feedback
rdner Mar 21, 2024
e212d9a
Merge branch 'main' into fetch-versions-from-public-source
rdner Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .agent-versions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
{
"testVersions": [
"8.14.0-SNAPSHOT",
"8.13.0-SNAPSHOT",
"8.12.3-SNAPSHOT",
"8.12.2",
"8.12.1",
"7.17.19-SNAPSHOT",
"7.17.18"
]
}
32 changes: 27 additions & 5 deletions magefile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ import (
"github.com/elastic/elastic-agent/pkg/testing/multipass"
"github.com/elastic/elastic-agent/pkg/testing/ogc"
"github.com/elastic/elastic-agent/pkg/testing/runner"
"github.com/elastic/elastic-agent/pkg/testing/tools"
"github.com/elastic/elastic-agent/pkg/testing/tools/git"
pv "github.com/elastic/elastic-agent/pkg/testing/tools/product_versions"
"github.com/elastic/elastic-agent/pkg/testing/tools/snapshots"
"github.com/elastic/elastic-agent/pkg/version"
"github.com/elastic/elastic-agent/testing/upgradetest"
bversion "github.com/elastic/elastic-agent/version"
Expand Down Expand Up @@ -1576,17 +1578,37 @@ func (Integration) Single(ctx context.Context, testName string) error {
// UpdateVersions runs an update on the `.agent-versions.json` fetching
// the latest version list from the artifact API.
func (Integration) UpdateVersions(ctx context.Context) error {
// test 2 current 8.x version, 1 previous 7.x version and 1 recent snapshot
maxSnapshots := 3

branches, err := git.GetReleaseBranches(ctx)
if err != nil {
return fmt.Errorf("failed to list release branches: %w", err)
}

// -1 because we manually add 7.17 below
if len(branches) > maxSnapshots-1 {
branches = branches[:maxSnapshots-1]
}

// it's not a part of this repository, cannot be retrieved with `GetReleaseBranches`
branches = append(branches, "7.17")

// uncomment if want to have the current version snapshot on the list as well
// branches = append([]string{"master"}, branches...)
Comment on lines +1596 to +1597
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not necessary to have the current version snapshot on this list because we have plenty of tests that use it for upgrade/downgrade already. So, I skipped it to make the tests run faster.


reqs := upgradetest.VersionRequirements{
UpgradeToVersion: bversion.Agent,
CurrentMajors: 2,
PreviousMinors: 1,
PreviousMajors: 1,
RecentSnapshots: 1,
SnapshotBranches: branches,
}
b, _ := json.MarshalIndent(reqs, "", " ")
fmt.Printf("Current version requirements: \n%s\n", b)

aac := tools.NewArtifactAPIClient(tools.WithLogFunc(log.Default().Printf))
versions, err := upgradetest.FetchUpgradableVersions(ctx, aac, reqs)
pvc := pv.NewProductVersionsClient()
sc := snapshots.NewSnapshotsClient()
versions, err := upgradetest.FetchUpgradableVersions(ctx, pvc, sc, reqs)
if err != nil {
return fmt.Errorf("failed to fetch upgradable versions: %w", err)
}
Expand Down
100 changes: 8 additions & 92 deletions pkg/testing/tools/artifacts_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,34 +9,24 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"time"

"github.com/elastic/elastic-agent/pkg/version"
)

const (
defaultArtifactAPIURL = "https://artifacts-api.elastic.co/"

artifactsAPIV1VersionsEndpoint = "v1/versions/"
artifactsAPIV1VersionBuildsEndpoint = "v1/versions/%s/builds/"
artifactAPIV1BuildDetailsEndpoint = "v1/versions/%s/builds/%s"
// artifactAPIV1SearchVersionPackage = "v1/search/%s/%s"

artifactElasticAgentProject = "elastic-agent-package"
maxAttemptsForArtifactsAPICall = 6
retryIntervalForArtifactsAPICall = 5 * time.Second
)

var (
ErrLatestVersionNil = errors.New("latest version is nil")
ErrSnapshotVersionsEmpty = errors.New("snapshot list is nil")
ErrInvalidVersionRetrieved = errors.New("invalid version retrieved from artifact API")
ErrBuildNotFound = errors.New("there are no build that satisfy given conditions")

ErrBuildNotFound = errors.New("there are no builds that satisfy given conditions")
ErrBadHTTPStatusCode = errors.New("bad http status code")
)

Expand All @@ -45,12 +35,6 @@ type Manifests struct {
SecondsSinceLastUpdate int `json:"seconds-since-last-update"`
}

type VersionList struct {
Versions []string `json:"versions"`
Aliases []string `json:"aliases"`
Manifests Manifests `json:"manifests"`
}

type VersionBuilds struct {
Builds []string `json:"builds"`
Manifests Manifests `json:"manifests"`
Expand Down Expand Up @@ -107,11 +91,6 @@ type BuildDetails struct {
Manifests Manifests `json:"manifests"`
}

type SearchPackageResult struct {
Packages map[string]Package `json:"packages"`
Manifests Manifests `json:"manifests"`
}

type httpDoer interface {
Do(req *http.Request) (*http.Response, error)
}
Expand Down Expand Up @@ -156,26 +135,10 @@ func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
return c
}

// GetVersions returns a list of versions as server by the Artifact API along with some aliases and manifest information
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand that this function and the types above are not used by the new version of testing code but they are still existing operations and types of Artifact API, I would prefer to leave them there

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The artifact API is deprecated and we must eventually get rid of it entirely. Deleting these functions will make sure we're not using them again.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the datamodel is shared with the DRA manifests we need to make sure we don't delete too much.
As for this client, it was not about what we have to use in the tests, just a simple client that made accessing the snapshot API easier regardless of how it has been used. For the deprecation a simple Deprecated: comment on the client itself would suffice.

func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList, err error) {
joinedURL, err := aac.composeURL(artifactsAPIV1VersionsEndpoint)
if err != nil {
return nil, err
}

resp, err := aac.createAndPerformRequest(ctx, joinedURL)
if err != nil {
return nil, fmt.Errorf("getting versions: %w", err)
}

defer resp.Body.Close()
return checkResponseAndUnmarshal[VersionList](resp)
}

// GetBuildsForVersion returns a list of builds for a specific version.
// version should be one of the version strings returned by the GetVersions (expected format is semver
// with optional prerelease but no build metadata, for example 8.9.0-SNAPSHOT)
func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version string) (builds *VersionBuilds, err error) {
func (aac ArtifactAPIClient) getBuildsForVersion(ctx context.Context, version string) (builds *VersionBuilds, err error) {
joinedURL, err := aac.composeURL(fmt.Sprintf(artifactsAPIV1VersionBuildsEndpoint, version))
if err != nil {
return nil, err
Expand All @@ -196,15 +159,15 @@ func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version st
// Setting `offset` to 0 includes all builds, 1 skips the latest, and so forth.
// If there are no builds matching these conditions, returns `ErrBuildNotFound`.
func (aac ArtifactAPIClient) FindBuild(ctx context.Context, version, excludeHash string, offset int) (buildDetails *BuildDetails, err error) {
resp, err := aac.GetBuildsForVersion(ctx, version)
resp, err := aac.getBuildsForVersion(ctx, version)
if err != nil {
return nil, fmt.Errorf("failed to get a list of builds: %w", err)
}
if len(resp.Builds) < offset+1 {
return nil, ErrBuildNotFound
}
for _, buildID := range resp.Builds[offset:] {
details, err := aac.GetBuildDetails(ctx, version, buildID)
details, err := aac.getBuildDetails(ctx, version, buildID)
if err != nil {
return nil, fmt.Errorf("failed to get build information for %q: %w", buildID, err)
}
Expand All @@ -219,7 +182,7 @@ func (aac ArtifactAPIClient) FindBuild(ctx context.Context, version, excludeHash
// GetBuildDetails returns the list of project and artifacts related to a specific build.
// Version parameter format follows semver (without build metadata) and buildID format is <major>.<minor>.<patch>-<buildhash> as returned by
// GetBuildsForVersion()
func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string, buildID string) (buildDetails *BuildDetails, err error) {
func (aac ArtifactAPIClient) getBuildDetails(ctx context.Context, version string, buildID string) (buildDetails *BuildDetails, err error) {
joinedURL, err := aac.composeURL(fmt.Sprintf(artifactAPIV1BuildDetailsEndpoint, version, buildID))
if err != nil {
return nil, err
Expand Down Expand Up @@ -291,59 +254,12 @@ func checkResponseAndUnmarshal[T any](resp *http.Response) (*T, error) {
return nil, fmt.Errorf("%d: %w", resp.StatusCode, ErrBadHTTPStatusCode)
}

respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading response body: %w", err)
}
d := json.NewDecoder(resp.Body)
result := new(T)
err = json.Unmarshal(respBytes, result)

err := d.Decode(&result)
if err != nil {
return nil, fmt.Errorf("unmarshaling: %w", err)
return nil, fmt.Errorf("failed to parse response: %w", err)
}

return result, nil
}

func (aac ArtifactAPIClient) GetLatestSnapshotVersion(ctx context.Context) (*version.ParsedSemVer, error) {
vList, err := aac.GetVersions(ctx)
if err != nil {
return nil, err
}

if vList == nil {
return nil, ErrSnapshotVersionsEmpty
}

sortedParsedVersions := make(version.SortableParsedVersions, 0, len(vList.Versions))
for _, v := range vList.Versions {
pv, err := version.ParseVersion(v)
if err != nil {
aac.logFunc("invalid version retrieved from artifact API: %q", v)
return nil, ErrInvalidVersionRetrieved
}
sortedParsedVersions = append(sortedParsedVersions, pv)
}

if len(sortedParsedVersions) == 0 {
return nil, ErrSnapshotVersionsEmpty
}

// normally the output of the versions returned by artifact API is already
// sorted in ascending order.If we want to sort in descending order we need
// to pass a sort.Reverse to sort.Sort.
sort.Sort(sort.Reverse(sortedParsedVersions))

var latestSnapshotVersion *version.ParsedSemVer
// fetch the latest SNAPSHOT build
for _, pv := range sortedParsedVersions {
if pv.IsSnapshot() {
latestSnapshotVersion = pv
break
}
}
if latestSnapshotVersion == nil {
return nil, ErrLatestVersionNil
}
return latestSnapshotVersion, nil
}
Loading
Loading