Skip to content

Commit f25124a

Browse files
rdnerrdner
authored
Don't use artifact API for snapshot downloads (#4693)
* The Elastic Agent upgrade to a snapshot is now using the more reliable snapshot API * The artifact fetcher in testing is now using the same snapshot API The artifact API is now used only in a single integration test-case `TestStandaloneDowngradeToSpecificSnapshotBuild`.
1 parent 13a4157 commit f25124a

File tree

6 files changed

+106
-260
lines changed

6 files changed

+106
-260
lines changed

‎changelog/fragments/1715158488-use-snapshot-api.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
kind: enhancement
2+
summary: Use more stable snapshot API for upgrades to snapshot versions
3+
component: "elastic-agent"
4+
pr: https://github.com/elastic/elastic-agent/pull/4693
5+
issue: https://github.com/elastic/elastic-agent/issues/4458

‎internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader.go

+29-39
Original file line numberDiff line numberDiff line change
@@ -124,60 +124,50 @@ func snapshotURI(ctx context.Context, client *gohttp.Client, versionOverride *ag
124124
version = versionOverride.CoreVersion()
125125
}
126126

127-
artifactsURI := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", version)
128-
request, err := gohttp.NewRequestWithContext(ctx, gohttp.MethodGet, artifactsURI, nil)
127+
// otherwise, if we don't know the exact build and we're trying to find the latest snapshot build
128+
buildID, err := findLatestSnapshot(ctx, client, version)
129129
if err != nil {
130-
return "", fmt.Errorf("creating request to artifact api: %w", err)
130+
return "", fmt.Errorf("failed to find snapshot information for version %q: %w", version, err)
131131
}
132132

133-
resp, err := client.Do(request)
133+
return fmt.Sprintf(snapshotURIFormat, version, buildID), nil
134+
}
135+
136+
func findLatestSnapshot(ctx context.Context, client *gohttp.Client, version string) (buildID string, err error) {
137+
latestSnapshotURI := fmt.Sprintf("https://snapshots.elastic.co/latest/%s-SNAPSHOT.json", version)
138+
request, err := gohttp.NewRequestWithContext(ctx, gohttp.MethodGet, latestSnapshotURI, nil)
134139
if err != nil {
135-
return "", err
140+
return "", fmt.Errorf("failed to create request to the snapshot API: %w", err)
136141
}
137-
defer resp.Body.Close()
138-
139-
body := struct {
140-
Packages map[string]interface{} `json:"packages"`
141-
}{}
142142

143-
dec := json.NewDecoder(resp.Body)
144-
if err := dec.Decode(&body); err != nil {
143+
resp, err := client.Do(request)
144+
if err != nil {
145145
return "", err
146146
}
147+
defer resp.Body.Close()
147148

148-
if len(body.Packages) == 0 {
149-
return "", fmt.Errorf("no packages found in snapshot repo")
150-
}
149+
switch resp.StatusCode {
150+
case gohttp.StatusNotFound:
151+
return "", fmt.Errorf("snapshot for version %q not found", version)
151152

152-
for k, pkg := range body.Packages {
153-
pkgMap, ok := pkg.(map[string]interface{})
154-
if !ok {
155-
return "", fmt.Errorf("content of '%s' is not a map", k)
153+
case gohttp.StatusOK:
154+
var info struct {
155+
BuildID string `json:"build_id"`
156156
}
157157

158-
uriVal, found := pkgMap["url"]
159-
if !found {
160-
return "", fmt.Errorf("item '%s' does not contain url", k)
158+
dec := json.NewDecoder(resp.Body)
159+
if err := dec.Decode(&info); err != nil {
160+
return "", err
161161
}
162162

163-
uri, ok := uriVal.(string)
164-
if !ok {
165-
return "", fmt.Errorf("uri is not a string")
163+
parts := strings.Split(info.BuildID, "-")
164+
if len(parts) != 2 {
165+
return "", fmt.Errorf("wrong format for a build ID: %s", info.BuildID)
166166
}
167167

168-
// Because we're iterating over a map from the API response,
169-
// the order is random and some elements there do not contain the
170-
// `/beats/elastic-agent/` substring, so we need to go through the
171-
// whole map before returning an error.
172-
//
173-
// One of the elements that might be there and do not contain this
174-
// substring is the `elastic-agent-shipper`, whose URL is something like:
175-
// https://snapshots.elastic.co/8.7.0-d050210c/downloads/elastic-agent-shipper/elastic-agent-shipper-8.7.0-SNAPSHOT-linux-x86_64.tar.gz
176-
index := strings.Index(uri, "/beats/elastic-agent/")
177-
if index != -1 {
178-
return uri[:index], nil
179-
}
180-
}
168+
return parts[1], nil
181169

182-
return "", fmt.Errorf("uri not detected")
170+
default:
171+
return "", fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, latestSnapshotURI)
172+
}
183173
}

‎internal/pkg/agent/application/upgrade/artifact/download/snapshot/downloader_test.go

+23-115
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"net"
1313
"net/http"
1414
"net/http/httptest"
15+
"os"
1516
"path/filepath"
1617
"testing"
1718

@@ -37,91 +38,30 @@ func TestNonDefaultSourceURI(t *testing.T) {
3738

3839
}
3940

40-
const artifactAPIElasticAgentSearchResponse = `
41-
{
42-
"packages": {
43-
"elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz": {
44-
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz",
45-
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz.sha512",
46-
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-aarch64.tar.gz.asc",
47-
"type": "tar",
48-
"architecture": "aarch64",
49-
"os": [
50-
"darwin"
51-
]
52-
},
53-
"elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip": {
54-
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip",
55-
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip.sha512",
56-
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-windows-x86_64.zip.asc",
57-
"type": "zip",
58-
"architecture": "x86_64",
59-
"os": [
60-
"windows"
61-
]
62-
},
63-
"elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz": {
64-
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz",
65-
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz.sha512",
66-
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/elastic-agent-core/elastic-agent-core-1.2.3-SNAPSHOT-linux-arm64.tar.gz.asc",
67-
"type": "tar",
68-
"architecture": "arm64",
69-
"os": [
70-
"linux"
71-
]
72-
},
73-
"elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": {
74-
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz",
75-
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512",
76-
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.asc",
77-
"type": "tar",
78-
"architecture": "x86_64",
79-
"os": [
80-
"linux"
81-
]
82-
},
83-
"elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz": {
84-
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz",
85-
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz.sha512",
86-
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-arm64.tar.gz.asc",
87-
"type": "tar",
88-
"architecture": "arm64",
89-
"os": [
90-
"linux"
91-
]
92-
},
93-
"elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz": {
94-
"url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz",
95-
"sha_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz.sha512",
96-
"asc_url": "https://snapshots.elastic.co/1.2.3-33e8d7e1/downloads/beats/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-darwin-x86_64.tar.gz.asc",
97-
"type": "tar",
98-
"architecture": "x86_64",
99-
"os": [
100-
"darwin"
101-
]
102-
}
103-
},
104-
"manifests": {
105-
"last-update-time": "Tue, 05 Dec 2023 15:47:06 UTC",
106-
"seconds-since-last-update": 201
107-
}
108-
}
109-
`
110-
11141
var agentSpec = artifact.Artifact{
11242
Name: "Elastic Agent",
11343
Cmd: "elastic-agent",
11444
Artifact: "beat/elastic-agent",
11545
}
11646

117-
type downloadHttpResponse struct {
118-
statusCode int
119-
headers http.Header
120-
Body []byte
47+
func readFile(t *testing.T, name string) []byte {
48+
bytes, err := os.ReadFile(name)
49+
require.NoError(t, err)
50+
51+
return bytes
12152
}
12253

12354
func TestDownloadVersion(t *testing.T) {
124-
55+
files := map[string][]byte{
56+
// links for the latest snapshot
57+
"/latest/8.14.0-SNAPSHOT.json": readFile(t, "./testdata/latest-snapshot.json"),
58+
"/8.14.0-6d69ee76/downloads/beat/elastic-agent/elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz": {},
59+
"/8.14.0-6d69ee76/downloads/beat/elastic-agent/elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz.sha512": {},
60+
61+
// links for a specific build
62+
"/8.13.3-76ce1a63/downloads/beat/elastic-agent/elastic-agent-8.13.3-SNAPSHOT-linux-x86_64.tar.gz": {},
63+
"/8.13.3-76ce1a63/downloads/beat/elastic-agent/elastic-agent-8.13.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": {},
64+
}
12565
type fields struct {
12666
config *artifact.Config
12767
}
@@ -131,59 +71,33 @@ func TestDownloadVersion(t *testing.T) {
13171
}
13272
tests := []struct {
13373
name string
134-
files map[string]downloadHttpResponse
13574
fields fields
13675
args args
13776
want string
13877
wantErr assert.ErrorAssertionFunc
13978
}{
14079
{
14180
name: "happy path snapshot version",
142-
files: map[string]downloadHttpResponse{
143-
"/1.2.3-33e8d7e1/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": {
144-
statusCode: http.StatusOK,
145-
Body: []byte("This is a fake linux elastic agent archive"),
146-
},
147-
"/1.2.3-33e8d7e1/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": {
148-
statusCode: http.StatusOK,
149-
Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"),
150-
},
151-
"/v1/search/1.2.3-SNAPSHOT/elastic-agent": {
152-
statusCode: http.StatusOK,
153-
headers: map[string][]string{"Content-Type": {"application/json"}},
154-
Body: []byte(artifactAPIElasticAgentSearchResponse),
155-
},
156-
},
15781
fields: fields{
15882
config: &artifact.Config{
15983
OperatingSystem: "linux",
16084
Architecture: "64",
16185
},
16286
},
163-
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "")},
164-
want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz",
87+
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(8, 14, 0, "SNAPSHOT", "")},
88+
want: "elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz",
16589
wantErr: assert.NoError,
16690
},
16791
{
16892
name: "happy path snapshot version with build metadata",
169-
files: map[string]downloadHttpResponse{
170-
"/1.2.3-buildid/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz": {
171-
statusCode: http.StatusOK,
172-
Body: []byte("This is a fake linux elastic agent archive"),
173-
},
174-
"/1.2.3-buildid/downloads/beat/elastic-agent/elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz.sha512": {
175-
statusCode: http.StatusOK,
176-
Body: []byte("somesha512 elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz"),
177-
},
178-
},
17993
fields: fields{
18094
config: &artifact.Config{
18195
OperatingSystem: "linux",
18296
Architecture: "64",
18397
},
18498
},
185-
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(1, 2, 3, "SNAPSHOT", "buildid")},
186-
want: "elastic-agent-1.2.3-SNAPSHOT-linux-x86_64.tar.gz",
99+
args: args{a: agentSpec, version: agtversion.NewParsedSemVer(8, 13, 3, "SNAPSHOT", "76ce1a63")},
100+
want: "elastic-agent-8.13.3-SNAPSHOT-linux-x86_64.tar.gz",
187101
wantErr: assert.NoError,
188102
},
189103
}
@@ -195,21 +109,15 @@ func TestDownloadVersion(t *testing.T) {
195109

196110
handleDownload := func(rw http.ResponseWriter, req *http.Request) {
197111
path := req.URL.Path
112+
t.Logf("incoming request for %s", path)
198113

199-
resp, ok := tt.files[path]
114+
file, ok := files[path]
200115
if !ok {
201116
rw.WriteHeader(http.StatusNotFound)
202117
return
203118
}
204119

205-
for k, values := range resp.headers {
206-
for _, v := range values {
207-
rw.Header().Set(k, v)
208-
}
209-
}
210-
211-
rw.WriteHeader(resp.statusCode)
212-
_, err := io.Copy(rw, bytes.NewReader(resp.Body))
120+
_, err := io.Copy(rw, bytes.NewReader(file))
213121
assert.NoError(t, err, "error writing out response body")
214122
}
215123
server := httptest.NewTLSServer(http.HandlerFunc(handleDownload))

‎internal/pkg/agent/application/upgrade/artifact/download/snapshot/testdata/latest-snapshot.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"version": "8.14.0-SNAPSHOT",
3+
"build_id": "8.14.0-6d69ee76",
4+
"manifest_url": "https://snapshots.elastic.co/8.14.0-6d69ee76/manifest-8.14.0-SNAPSHOT.json",
5+
"summary_url": "https://snapshots.elastic.co/8.14.0-6d69ee76/summary-8.14.0-SNAPSHOT.html"
6+
}

0 commit comments

Comments
 (0)