Skip to content

Commit 01c6fd2

Browse files
committed
manifest,sources: add librepo support to Serialize/GenSources
This commit enables librepo sources generation via a new osbuild.RpmDownloader iota that is passed to to `manifest.Serialize()` and `osbuild.GenSources()`. We also need to pass the resolved repoConfig to `manifest.Serialize()`. This is currently not type-safe, ideally we would look into how to do this in a type-safe way. Serialize should also take less args ideally.
1 parent e0aeb90 commit 01c6fd2

File tree

9 files changed

+166
-43
lines changed

9 files changed

+166
-43
lines changed

cmd/build/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ func makeManifest(
8282
return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %w", err)
8383
}
8484

85-
mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil)
85+
mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs, 0)
8686
if err != nil {
8787
return nil, fmt.Errorf("[ERROR] manifest serialization failed: %w", err)
8888
}

cmd/gen-manifests/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ func makeManifestJob(
251251
commitSpecs = mockResolveCommits(manifest.GetOSTreeSourceSpecs())
252252
}
253253

254-
mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs)
254+
mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs, 0)
255255
if err != nil {
256256
return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error())
257257
}

cmd/osbuild-playground/playground.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos
4949
fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error())
5050
}
5151

52-
bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil)
52+
bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil, 0)
5353
if err != nil {
5454
panic("failed to serialize manifest: " + err.Error())
5555
}

pkg/distro/distro_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func TestImageTypePipelineNames(t *testing.T) {
134134
}
135135
commits[name] = commitSpecs
136136
}
137-
mf, err := m.Serialize(packageSets, containers, commits, repoSets)
137+
mf, err := m.Serialize(packageSets, containers, commits, repoSets, 0)
138138
assert.NoError(err)
139139
pm := new(manifest)
140140
err = json.Unmarshal(mf, pm)

pkg/image/bootc_disk_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func makeBootcDiskImageOsbuildManifest(t *testing.T, opts *bootcDiskImageTestOpt
8888
"image": []container.Spec{{Source: "other-src", Digest: makeFakeDigest(t), ImageID: makeFakeDigest(t)}},
8989
}
9090

91-
osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil)
91+
osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil, 0)
9292
require.Nil(t, err)
9393

9494
return osbuildManifest

pkg/image/installer_image_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ func instantiateAndSerialize(t *testing.T, img image.ImageKind, packages map[str
364364
_, err := img.InstantiateManifest(&mf, nil, &runner.CentOS{Version: 9}, rng)
365365
assert.NoError(t, err)
366366

367-
mfs, err := mf.Serialize(packages, containers, commits, nil)
367+
mfs, err := mf.Serialize(packages, containers, commits, nil, 0)
368368
assert.NoError(t, err)
369369

370370
return string(mfs)

pkg/manifest/manifest.go

+13-18
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,7 @@ const (
4141
DISTRO_FEDORA
4242
)
4343

44-
type Inputs struct {
45-
Packages []rpmmd.PackageSpec
46-
Containers []container.Spec
47-
Commits []ostree.CommitSpec
48-
RpmRepos []rpmmd.RepoConfig
49-
}
44+
type Inputs osbuild.SourceInputs
5045

5146
// An OSBuildManifest is an opaque JSON object, which is a valid input to osbuild
5247
type OSBuildManifest []byte
@@ -149,32 +144,32 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec {
149144
// only depsolved PackageSpecs/RepoConfigs are passed so that we
150145
// have a valid mapping of pkg.RepoID<->repo.Id which will be important
151146
// for librepo
152-
func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) {
153-
pipelines := make([]osbuild.Pipeline, 0)
154-
packages := make([]rpmmd.PackageSpec, 0)
155-
commits := make([]ostree.CommitSpec, 0)
156-
inline := make([]string, 0)
157-
containers := make([]container.Spec, 0)
147+
func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, resolvedRpmRepos map[string][]rpmmd.RepoConfig, rpmDownloader osbuild.RpmDownloader) (OSBuildManifest, error) {
158148
for _, pipeline := range m.pipelines {
159149
pipeline.serializeStart(Inputs{
160150
Packages: packageSets[pipeline.Name()],
161151
Containers: containerSpecs[pipeline.Name()],
162152
Commits: ostreeCommits[pipeline.Name()],
163-
RpmRepos: rpmRepos[pipeline.Name()],
153+
RpmRepos: resolvedRpmRepos[pipeline.Name()],
164154
})
165155
}
156+
157+
var pipelines []osbuild.Pipeline
158+
var mergedInputs osbuild.SourceInputs
166159
for _, pipeline := range m.pipelines {
167-
commits = append(commits, pipeline.getOSTreeCommits()...)
168160
pipelines = append(pipelines, pipeline.serialize())
169-
packages = append(packages, packageSets[pipeline.Name()]...)
170-
inline = append(inline, pipeline.getInline()...)
171-
containers = append(containers, pipeline.getContainerSpecs()...)
161+
162+
mergedInputs.Commits = append(mergedInputs.Commits, pipeline.getOSTreeCommits()...)
163+
mergedInputs.Packages = append(mergedInputs.Packages, packageSets[pipeline.Name()]...)
164+
mergedInputs.RpmRepos = append(mergedInputs.RpmRepos, resolvedRpmRepos[pipeline.Name()]...)
165+
mergedInputs.Containers = append(mergedInputs.Containers, pipeline.getContainerSpecs()...)
166+
mergedInputs.InlineData = append(mergedInputs.InlineData, pipeline.getInline()...)
172167
}
173168
for _, pipeline := range m.pipelines {
174169
pipeline.serializeEnd()
175170
}
176171

177-
sources, err := osbuild.GenSources(packages, commits, inline, containers)
172+
sources, err := osbuild.GenSources(mergedInputs, rpmDownloader)
178173
if err != nil {
179174
return nil, err
180175
}

pkg/osbuild/source.go

+66-15
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,33 @@ package osbuild
33
import (
44
"encoding/json"
55
"errors"
6+
"fmt"
67

78
"github.com/osbuild/images/pkg/container"
89
"github.com/osbuild/images/pkg/ostree"
910
"github.com/osbuild/images/pkg/rpmmd"
1011
)
1112

13+
// RpmDownloader specifies what backend to use for rpm downloads
14+
// Note that the librepo backend requires a newer osbuild.
15+
type RpmDownloader uint64
16+
17+
const (
18+
RpmDownloaderCurl = iota
19+
RpmDownloaderLibrepo = iota
20+
)
21+
22+
// SourceInputs contains the inputs to generate osbuild.Sources
23+
// Note that for Packages/RpmRepos the depsolve resolved results
24+
// must be passed
25+
type SourceInputs struct {
26+
Packages []rpmmd.PackageSpec
27+
Containers []container.Spec
28+
Commits []ostree.CommitSpec
29+
RpmRepos []rpmmd.RepoConfig
30+
InlineData []string
31+
}
32+
1233
// A Sources map contains all the sources made available to an osbuild run
1334
type Sources map[string]Source
1435

@@ -54,25 +75,55 @@ func (sources *Sources) UnmarshalJSON(data []byte) error {
5475
return nil
5576
}
5677

57-
func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec) (Sources, error) {
78+
func addPackagesCurl(sources Sources, packages []rpmmd.PackageSpec) error {
79+
curl := NewCurlSource()
80+
for _, pkg := range packages {
81+
err := curl.AddPackage(pkg)
82+
if err != nil {
83+
return err
84+
}
85+
}
86+
sources["org.osbuild.curl"] = curl
87+
return nil
88+
}
89+
90+
func addPackagesLibrepo(sources Sources, packages []rpmmd.PackageSpec, rpmRepos []rpmmd.RepoConfig) error {
91+
librepo := NewLibrepoSource()
92+
for _, pkg := range packages {
93+
err := librepo.AddPackage(pkg, rpmRepos)
94+
if err != nil {
95+
return err
96+
}
97+
}
98+
sources["org.osbuild.librepo"] = librepo
99+
return nil
100+
}
101+
102+
// GenSources generates the Sources from the given inputs. Note that
103+
// the packages and rpmRepos need to come from the *resolved* set.
104+
func GenSources(inputs SourceInputs, rpmDownloader RpmDownloader) (Sources, error) {
58105
sources := Sources{}
59106

60107
// collect rpm package sources
61-
if len(packages) > 0 {
62-
curl := NewCurlSource()
63-
for _, pkg := range packages {
64-
err := curl.AddPackage(pkg)
65-
if err != nil {
66-
return nil, err
67-
}
108+
if len(inputs.Packages) > 0 {
109+
var err error
110+
switch rpmDownloader {
111+
case RpmDownloaderCurl:
112+
err = addPackagesCurl(sources, inputs.Packages)
113+
case RpmDownloaderLibrepo:
114+
err = addPackagesLibrepo(sources, inputs.Packages, inputs.RpmRepos)
115+
default:
116+
err = fmt.Errorf("unknown rpm downloader %v", rpmDownloader)
117+
}
118+
if err != nil {
119+
return nil, err
68120
}
69-
sources["org.osbuild.curl"] = curl
70121
}
71122

72123
// collect ostree commit sources
73-
if len(ostreeCommits) > 0 {
124+
if len(inputs.Commits) > 0 {
74125
ostree := NewOSTreeSource()
75-
for _, commit := range ostreeCommits {
126+
for _, commit := range inputs.Commits {
76127
ostree.AddItem(commit)
77128
}
78129
if len(ostree.Items) > 0 {
@@ -81,21 +132,21 @@ func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec,
81132
}
82133

83134
// collect inline data sources
84-
if len(inlineData) > 0 {
135+
if len(inputs.InlineData) > 0 {
85136
ils := NewInlineSource()
86-
for _, data := range inlineData {
137+
for _, data := range inputs.InlineData {
87138
ils.AddItem(data)
88139
}
89140

90141
sources["org.osbuild.inline"] = ils
91142
}
92143

93144
// collect skopeo and local container sources
94-
if len(containers) > 0 {
145+
if len(inputs.Containers) > 0 {
95146
skopeo := NewSkopeoSource()
96147
skopeoIndex := NewSkopeoIndexSource()
97148
localContainers := NewContainersStorageSource()
98-
for _, c := range containers {
149+
for _, c := range inputs.Containers {
99150
if c.LocalStorage {
100151
localContainers.AddItem(c.ImageID)
101152
} else {

pkg/osbuild/source_test.go

+81-4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/stretchr/testify/assert"
1010

1111
"github.com/osbuild/images/pkg/container"
12+
"github.com/osbuild/images/pkg/rpmmd"
1213
)
1314

1415
func TestSource_UnmarshalJSON(t *testing.T) {
@@ -119,7 +120,7 @@ func TestSource_UnmarshalJSON(t *testing.T) {
119120
}
120121

121122
func TestGenSourcesTrivial(t *testing.T) {
122-
sources, err := GenSources(nil, nil, nil, nil)
123+
sources, err := GenSources(SourceInputs{}, 0)
123124
assert.NoError(t, err)
124125

125126
jsonOutput, err := json.MarshalIndent(sources, "", " ")
@@ -135,7 +136,7 @@ func TestGenSourcesContainerStorage(t *testing.T) {
135136
LocalStorage: true,
136137
},
137138
}
138-
sources, err := GenSources(nil, nil, nil, containers)
139+
sources, err := GenSources(SourceInputs{Containers: containers}, 0)
139140
assert.NoError(t, err)
140141

141142
jsonOutput, err := json.MarshalIndent(sources, "", " ")
@@ -159,7 +160,7 @@ func TestGenSourcesSkopeo(t *testing.T) {
159160
ImageID: imageID,
160161
},
161162
}
162-
sources, err := GenSources(nil, nil, nil, containers)
163+
sources, err := GenSources(SourceInputs{Containers: containers}, 0)
163164
assert.NoError(t, err)
164165

165166
jsonOutput, err := json.MarshalIndent(sources, "", " ")
@@ -190,7 +191,7 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) {
190191
ImageID: imageID,
191192
},
192193
}
193-
sources, err := GenSources(nil, nil, nil, containers)
194+
sources, err := GenSources(SourceInputs{Containers: containers}, 0)
194195
assert.NoError(t, err)
195196

196197
jsonOutput, err := json.MarshalIndent(sources, "", " ")
@@ -217,3 +218,79 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) {
217218
}
218219
}`)
219220
}
221+
222+
// TODO: move into a common "rpmtest" package
223+
var opensslPkg = rpmmd.PackageSpec{
224+
Name: "openssl-libs",
225+
RemoteLocation: "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
226+
Checksum: "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666",
227+
Path: "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
228+
RepoID: "repo_id_metalink",
229+
}
230+
231+
var fakeRepos = []rpmmd.RepoConfig{
232+
{
233+
Id: "repo_id_metalink",
234+
Metalink: "http://example.com/metalink",
235+
},
236+
}
237+
238+
func TestGenSourcesRpmWithLibcurl(t *testing.T) {
239+
inputs := SourceInputs{
240+
Packages: []rpmmd.PackageSpec{opensslPkg},
241+
RpmRepos: fakeRepos,
242+
}
243+
sources, err := GenSources(inputs, RpmDownloaderCurl)
244+
assert.NoError(t, err)
245+
246+
jsonOutput, err := json.MarshalIndent(sources, "", " ")
247+
assert.NoError(t, err)
248+
assert.Equal(t, string(jsonOutput), `{
249+
"org.osbuild.curl": {
250+
"items": {
251+
"sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
252+
"url": "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm"
253+
}
254+
}
255+
}
256+
}`)
257+
}
258+
259+
func TestGenSourcesRpmWithLibrepo(t *testing.T) {
260+
inputs := SourceInputs{
261+
Packages: []rpmmd.PackageSpec{opensslPkg},
262+
RpmRepos: fakeRepos,
263+
}
264+
sources, err := GenSources(inputs, RpmDownloaderLibrepo)
265+
assert.NoError(t, err)
266+
267+
jsonOutput, err := json.MarshalIndent(sources, "", " ")
268+
assert.NoError(t, err)
269+
assert.Equal(t, string(jsonOutput), `{
270+
"org.osbuild.librepo": {
271+
"items": {
272+
"sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
273+
"path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
274+
"mirror": "repo_id_metalink"
275+
}
276+
},
277+
"options": {
278+
"mirrors": {
279+
"repo_id_metalink": {
280+
"url": "http://example.com/metalink",
281+
"type": "metalink"
282+
}
283+
}
284+
}
285+
}
286+
}`)
287+
}
288+
289+
func TestGenSourcesRpmBad(t *testing.T) {
290+
inputs := SourceInputs{
291+
Packages: []rpmmd.PackageSpec{opensslPkg},
292+
RpmRepos: fakeRepos,
293+
}
294+
_, err := GenSources(inputs, 99)
295+
assert.EqualError(t, err, "unknown rpm downloader 99")
296+
}

0 commit comments

Comments
 (0)