Skip to content

Commit 0e7a211

Browse files
dwhyrockcmacknzpierrehilbert
authored
[Ind Agent] Update packaging to properly package from manifest if given (#4885)
* Download from manifest if version has +1 patch * Try modified globExpr * use version package * catch err * use panic * Add the filepath * better package finding * more intermediate work * more progress * it seems to maybe work * fixed bug * Copying spec files as well * temp test value * fixing linting errors * Clean up manifest code * Cleaning up * Cleanup 2 * removing test variable * addressing PR comments and cleanup * Adding manifest tests * Update magefile.go Co-authored-by: Craig MacKenzie <craig.mackenzie@elastic.co> * Switch to go:embed for tests. * Build component specs from external binaries. * Convert component to project in var names * Return error when package not found. Add contexts where necessary * Filter unsupported platforms. * Fix darwin/arm64 build. * Several renames for consistency. * A few more renames. * Move code out of magefile * mage fmt * Fix log message. * Fix lint warnings. * Rename test. * Refactor to share download from manifest logic. --------- Co-authored-by: Craig MacKenzie <craig.mackenzie@elastic.co> Co-authored-by: Pierre HILBERT <pierre.hilbert@elastic.co>
1 parent c46a379 commit 0e7a211

8 files changed

+18459
-216
lines changed

‎dev-tools/mage/checksums.go

+281
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package mage
6+
7+
import (
8+
"fmt"
9+
"log"
10+
"os"
11+
"path/filepath"
12+
"strings"
13+
14+
"github.com/magefile/mage/mg"
15+
"github.com/otiai10/copy"
16+
17+
"github.com/elastic/elastic-agent/dev-tools/mage/manifest"
18+
"github.com/elastic/elastic-agent/pkg/testing/tools"
19+
)
20+
21+
const ComponentSpecFileSuffix = ".spec.yml"
22+
23+
func CopyComponentSpecs(componentName, versionedDropPath string) (string, error) {
24+
specFileName := componentName + ComponentSpecFileSuffix
25+
targetPath := filepath.Join(versionedDropPath, specFileName)
26+
27+
if _, err := os.Stat(targetPath); err != nil {
28+
fmt.Printf(">> File %s does not exist, reverting to local specfile\n", targetPath)
29+
// spec not present copy from local
30+
sourceSpecFile := filepath.Join("specs", specFileName)
31+
if mg.Verbose() {
32+
log.Printf("Copy spec from %s to %s", sourceSpecFile, targetPath)
33+
}
34+
err := Copy(sourceSpecFile, targetPath)
35+
if err != nil {
36+
return "", fmt.Errorf("failed copying spec file %q to %q: %w", sourceSpecFile, targetPath, err)
37+
}
38+
}
39+
40+
// compute checksum
41+
return GetSHA512Hash(targetPath)
42+
}
43+
44+
// This is a helper function for flattenDependencies that's used when not packaging from a manifest
45+
func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string {
46+
globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion))
47+
if mg.Verbose() {
48+
log.Printf("Finding files to copy with %s", globExpr)
49+
}
50+
files, err := filepath.Glob(globExpr)
51+
if err != nil {
52+
panic(err)
53+
}
54+
if mg.Verbose() {
55+
log.Printf("Validating checksums for %+v", files)
56+
log.Printf("--- Copying into %s: %v", versionedDropPath, files)
57+
}
58+
59+
checksums := make(map[string]string)
60+
for _, f := range files {
61+
options := copy.Options{
62+
OnSymlink: func(_ string) copy.SymlinkAction {
63+
return copy.Shallow
64+
},
65+
Sync: true,
66+
}
67+
if mg.Verbose() {
68+
log.Printf("> prepare to copy %s into %s ", f, versionedDropPath)
69+
}
70+
71+
err = copy.Copy(f, versionedDropPath, options)
72+
if err != nil {
73+
panic(err)
74+
}
75+
76+
// copy spec file for match
77+
specName := filepath.Base(f)
78+
idx := strings.Index(specName, "-"+packageVersion)
79+
if idx != -1 {
80+
specName = specName[:idx]
81+
}
82+
if mg.Verbose() {
83+
log.Printf(">>>> Looking to copy spec file: [%s]", specName)
84+
}
85+
86+
checksum, err := CopyComponentSpecs(specName, versionedDropPath)
87+
if err != nil {
88+
panic(err)
89+
}
90+
91+
checksums[specName+ComponentSpecFileSuffix] = checksum
92+
}
93+
94+
return checksums
95+
}
96+
97+
// This is a helper function for flattenDependencies that's used when building from a manifest
98+
func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *tools.Build) map[string]string {
99+
checksums := make(map[string]string)
100+
if manifestResponse == nil {
101+
return checksums
102+
}
103+
104+
// Iterate over the component projects in the manifest
105+
projects := manifestResponse.Projects
106+
for componentName := range projects {
107+
// Iterate over the individual package files within each component project
108+
for pkgName := range projects[componentName].Packages {
109+
// Only care about packages that match the required package constraint (os/arch)
110+
if strings.Contains(pkgName, requiredPackage) {
111+
// Iterate over the external binaries that we care about for packaging agent
112+
for binary := range manifest.ExpectedBinaries {
113+
// If the individual package doesn't match the expected prefix, then continue
114+
if !strings.HasPrefix(pkgName, binary) {
115+
continue
116+
}
117+
118+
if mg.Verbose() {
119+
log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage)
120+
}
121+
122+
// Get the version from the component based on the version in the package name
123+
// This is useful in the case where it's an Independent Agent Release, where
124+
// the opted-in projects will be one patch version ahead of the rest of the
125+
// opted-out/previously-released projects
126+
componentVersion := getComponentVersion(componentName, requiredPackage, projects[componentName])
127+
if mg.Verbose() {
128+
log.Printf(">>>>>>> Component [%s]/[%s] version is [%s]", componentName, requiredPackage, componentVersion)
129+
}
130+
131+
// Combine the package name w/ the versioned flat path
132+
fullPath := filepath.Join(versionedFlatPath, pkgName)
133+
134+
// Eliminate the file extensions to get the proper directory
135+
// name that we need to copy
136+
var dirToCopy string
137+
if strings.HasSuffix(fullPath, ".tar.gz") {
138+
dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")]
139+
} else if strings.HasSuffix(fullPath, ".zip") {
140+
dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")]
141+
} else {
142+
dirToCopy = fullPath
143+
}
144+
if mg.Verbose() {
145+
log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy)
146+
}
147+
148+
// cloud-defend path exception
149+
// When untarred, cloud defend untars to:
150+
// cloud-defend-8.14.0-arm64
151+
// but the manifest (and most of this code) expects to be the same as
152+
// the name in the manifest, which is:
153+
// cloud-defend-8.14.0-linux-x86_64
154+
// So we have to do a bit of a transformation here
155+
if strings.Contains(dirToCopy, "cloud-defend") {
156+
if strings.Contains(dirToCopy, "x86_64") {
157+
dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "x86_64", "amd64")
158+
}
159+
if strings.Contains(dirToCopy, "arm64") {
160+
// Not actually replacing the arch, but removing the "linux"
161+
dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "arm64", "arm64")
162+
}
163+
if mg.Verbose() {
164+
log.Printf(">>>>>>> Adjusted cloud-defend directory to copy: [%s]", dirToCopy)
165+
}
166+
}
167+
168+
// Set copy options
169+
options := copy.Options{
170+
OnSymlink: func(_ string) copy.SymlinkAction {
171+
return copy.Shallow
172+
},
173+
Sync: true,
174+
}
175+
if mg.Verbose() {
176+
log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath)
177+
}
178+
179+
// Do the copy
180+
err := copy.Copy(dirToCopy, versionedDropPath, options)
181+
if err != nil {
182+
panic(err)
183+
}
184+
185+
// copy spec file for match
186+
specName := filepath.Base(dirToCopy)
187+
idx := strings.Index(specName, "-"+componentVersion)
188+
if idx != -1 {
189+
specName = specName[:idx]
190+
}
191+
if mg.Verbose() {
192+
log.Printf(">>>> Looking to copy spec file: [%s]", specName)
193+
}
194+
195+
checksum, err := CopyComponentSpecs(specName, versionedDropPath)
196+
if err != nil {
197+
panic(err)
198+
}
199+
200+
checksums[specName+ComponentSpecFileSuffix] = checksum
201+
}
202+
}
203+
}
204+
}
205+
206+
return checksums
207+
}
208+
209+
// This function is used when building with a Manifest. In that manifest, it's possible
210+
// for projects in an Independent Agent Release to have different versions since the opted-in
211+
// ones will be one patch version higher than the opted-out/previously released projects.
212+
// This function tries to find the versions from the package name
213+
func getComponentVersion(componentName string, requiredPackage string, componentProject tools.Project) string {
214+
var componentVersion string
215+
var foundIt bool
216+
// Iterate over all the packages in the component project
217+
for pkgName := range componentProject.Packages {
218+
// Only care about the external binaries that we want to package
219+
for binary, project := range manifest.ExpectedBinaries {
220+
// If the given component name doesn't match the external binary component, skip
221+
if componentName != project.Name {
222+
continue
223+
}
224+
225+
// Split the package name on the binary name prefix plus a dash
226+
firstSplit := strings.Split(pkgName, binary+"-")
227+
if len(firstSplit) < 2 {
228+
continue
229+
}
230+
231+
// Get the second part of the first split
232+
secondHalf := firstSplit[1]
233+
if len(secondHalf) < 2 {
234+
continue
235+
}
236+
237+
// Make sure the second half matches the required package
238+
if strings.Contains(secondHalf, requiredPackage) {
239+
// ignore packages with names where this splitting doesn't results in proper version
240+
if strings.Contains(secondHalf, "docker-image") {
241+
continue
242+
}
243+
if strings.Contains(secondHalf, "oss-") {
244+
continue
245+
}
246+
247+
// The component version should be the first entry after splitting w/ the requiredPackage
248+
componentVersion = strings.Split(secondHalf, "-"+requiredPackage)[0]
249+
foundIt = true
250+
// break out of inner loop
251+
break
252+
}
253+
}
254+
if foundIt {
255+
// break out of outer loop
256+
break
257+
}
258+
}
259+
260+
if componentVersion == "" {
261+
errMsg := fmt.Sprintf("Unable to determine component version for [%s]", componentName)
262+
panic(errMsg)
263+
}
264+
265+
return componentVersion
266+
}
267+
268+
// This is a helper function for the cloud-defend package.
269+
// When it is untarred, it does not have the same dirname as the package name.
270+
// This adjusts for that and returns the actual path on disk for cloud-defend
271+
func fixCloudDefendDirPath(dirPath string, componentVersion string, expectedArch string, actualArch string) string {
272+
fixedDirPath := dirPath
273+
274+
cloudDefendExpectedDirName := fmt.Sprintf("cloud-defend-%s-linux-%s", componentVersion, expectedArch)
275+
cloudDefendActualDirName := fmt.Sprintf("cloud-defend-%s-%s", componentVersion, actualArch)
276+
if strings.Contains(fixedDirPath, cloudDefendExpectedDirName) {
277+
fixedDirPath = strings.ReplaceAll(fixedDirPath, cloudDefendExpectedDirName, cloudDefendActualDirName)
278+
}
279+
280+
return fixedDirPath
281+
}

0 commit comments

Comments
 (0)