diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index bafa3391f2..2c2b883250 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -510,6 +510,10 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error { return err } + if err := phase.propagateArtifacts(ctx); err != nil { + return err + } + telemetry.GetTelemetryWerfIO().BuildFinished(ctx, true) return phase.createReport(ctx, imagesPairs) @@ -603,9 +607,8 @@ func (phase *BuildPhase) convergeImageSbom(ctx context.Context, name string, ima signerIdentity = phase.SbomSigningOptions.Signer().Fingerprint() } - finalStageDesc := phase.finalStageDescForImage(name, images) for _, img := range images { - if err := phase.convergePlatformImageSbom(ctx, name, img, finalStageDesc, signer, signerIdentity, breaker); err != nil { + if err := phase.convergePlatformImageSbom(ctx, name, img, signer, signerIdentity, breaker); err != nil { return err } } @@ -613,7 +616,7 @@ func (phase *BuildPhase) convergeImageSbom(ctx context.Context, name string, ima return nil } -func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name string, img *image.Image, finalStageDesc *imagePkg.StageDesc, signer signature.Signer, signerIdentity string, breaker *externalref.ResolverBreaker) error { +func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name string, img *image.Image, signer signature.Signer, signerIdentity string, breaker *externalref.ResolverBreaker) error { stageDesc := img.GetLastNonEmptyStageDesc() if stageDesc == nil { return fmt.Errorf("unable to converge sbom for image %q: stage descriptor is unavailable", name) @@ -677,25 +680,76 @@ func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name str return fmt.Errorf("unable to converge sbom for image %q: %w", name, err) } - if err := phase.sbomStep.PropagateArtifacts(ctx, name, stageDesc, finalStageDesc, phase.Conveyor.StorageManager.GetCacheStagesStorageList()); err != nil { - return fmt.Errorf("unable to propagate sbom for image %q: %w", name, err) - } - return nil } -// finalStageDescForImage returns the final repo descriptor to copy the SBOM artifacts into, or nil -// when there is nothing to copy. A single-platform image never has one: publishFinalImage stores the -// final repo descriptor in the content tag desc, which convergeImageSbom already uses as the SBOM -// target. Reaching for the last non-empty stage here instead panics, because an image resolved from -// the cache short-circuits in BeforeImageStages and never gets one. -func (phase *BuildPhase) finalStageDescForImage(name string, images []*image.Image) *imagePkg.StageDesc { - if len(images) == 1 { +// propagateArtifacts copies the artifacts attached to every image in the stages repo +// into the final repo and the cache repos. It runs after both SBOM and VEX +// convergence so it carries every attached artifact kind, and it runs on every build, +// so a destination holding the image without its artifacts is repaired by the next +// run. For a multi-platform image the per-platform artifacts are copied onto the +// platform manifest digests — preserved by the registry-level index copy — and the +// image-level artifacts (e.g. VEX) onto the index digest, which exists in the primary +// stages storage and in the final repo but never in a cache repo. +func (phase *BuildPhase) propagateArtifacts(ctx context.Context) error { + if _, isLocal := phase.Conveyor.StorageManager.GetStagesStorage().(*storage.LocalStagesStorage); isLocal { return nil } - if multiImg := phase.Conveyor.imagesTree.GetMultiplatformImage(name); multiImg != nil { - return multiImg.GetFinalStageDesc() + + cacheStagesStorageList := phase.Conveyor.StorageManager.GetCacheStagesStorageList() + + for _, pair := range phase.Conveyor.imagesTree.GetImagesByName(false) { + name, images := pair.Unpair() + + if multiImg := phase.Conveyor.imagesTree.GetMultiplatformImage(name); multiImg != nil { + finalDesc := multiImg.GetFinalStageDesc() + + for _, img := range images { + stageDesc := img.GetContentTagDesc() + if stageDesc == nil { + continue + } + + opts := PropagateArtifactsOptions{CacheStagesStorageList: cacheStagesStorageList} + if finalDesc != nil { + opts.FinalRepo = finalDesc.Info.Repository + opts.FinalDigest = stageDesc.Info.GetDigest() + } + + if err := phase.sbomStep.PropagateArtifacts(ctx, name, stageDesc.Info.Repository, stageDesc.Info.GetDigest(), opts); err != nil { + return fmt.Errorf("unable to propagate artifacts for image %q (platform %s): %w", name, img.TargetPlatform, err) + } + } + + if idxDesc := multiImg.GetStageDesc(); idxDesc != nil && finalDesc != nil { + opts := PropagateArtifactsOptions{ + FinalRepo: finalDesc.Info.Repository, + FinalDigest: finalDesc.Info.GetDigest(), + } + if err := phase.sbomStep.PropagateArtifacts(ctx, name, idxDesc.Info.Repository, idxDesc.Info.GetDigest(), opts); err != nil { + return fmt.Errorf("unable to propagate artifacts for image %q: %w", name, err) + } + } + + continue + } + + stageDesc := images[0].GetContentTagDesc() + if stageDesc == nil { + continue + } + + opts := PropagateArtifactsOptions{CacheStagesStorageList: cacheStagesStorageList} + if finalDesc := images[0].GetFinalContentTagDesc(); finalDesc != nil { + opts.FinalRepo = finalDesc.Info.Repository + opts.FinalDigest = finalDesc.Info.GetDigest() + } + + if err := phase.sbomStep.PropagateArtifacts(ctx, name, stageDesc.Info.Repository, stageDesc.Info.GetDigest(), opts); err != nil { + return fmt.Errorf("unable to propagate artifacts for image %q: %w", name, err) + } } + return nil } @@ -780,7 +834,7 @@ func (phase *BuildPhase) publishFinalImage(ctx context.Context, name string, img if err != nil { return fmt.Errorf("unable to copy image into final repo: %w", err) } - img.SetContentTagDesc(desc) + img.SetFinalContentTagDesc(desc) return nil } diff --git a/pkg/build/build_phase_propagate_test.go b/pkg/build/build_phase_propagate_test.go new file mode 100644 index 0000000000..9013084073 --- /dev/null +++ b/pkg/build/build_phase_propagate_test.go @@ -0,0 +1,239 @@ +package build + +import ( + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/build/image" + "github.com/werf/werf/v3/pkg/docker_registry" + imagePkg "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/oci/artifact" + "github.com/werf/werf/v3/pkg/storage" + "github.com/werf/werf/v3/pkg/storage/manager" + "github.com/werf/werf/v3/test/mock" +) + +var _ = Describe("BuildPhase propagateArtifacts", func() { + var ( + server *httptest.Server + stagesRepo string + finalRepo string + cacheRepo string + remoteOpts []remote.Option + ) + + pushImage := func(ctx SpecContext, repo, tag string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + ref, err := name.NewTag(repo + ":" + tag) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + return dgst.String() + } + + pushIndex := func(ctx SpecContext, repo, tag string, platforms []string) (string, []string) { + idx := v1.ImageIndex(empty.Index) + var children []string + + for _, platform := range platforms { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + parsed, err := v1.ParsePlatform(platform) + Expect(err).To(Succeed()) + + idx = mutate.AppendManifests(idx, mutate.IndexAddendum{Add: img, Descriptor: v1.Descriptor{Platform: parsed}}) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + children = append(children, dgst.String()) + } + + ref, err := name.NewTag(repo + ":" + tag) + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(ref, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := idx.Digest() + Expect(err).To(Succeed()) + return dgst.String(), children + } + + copyManifestByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + desc, err := remote.Get(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(toRef, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + return + } + + img, err := desc.Image() + Expect(err).To(Succeed()) + Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + attach := func(ctx SpecContext, repo, digest, payload string) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + Expect(store.Attach(ctx, digest, attestation.DSSEMediaType, []byte(payload), "checksum-"+payload, "", "")).To(Succeed()) + } + + expectAttached := func(ctx SpecContext, repo, digest, payload string) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + content, err := store.GetAttachedContent(ctx, digest, attestation.DSSEMediaType, nil) + Expect(err).To(Succeed(), "no artifact attached to %s in %s", digest, repo) + Expect(content).To(MatchJSON(payload)) + } + + expectNothingAttached := func(ctx SpecContext, repo, digest string) { + idx, err := artifact.PullFallbackIndex(ctx, repo, digest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + Expect(im.Manifests).To(BeEmpty(), "unexpected artifact attached to %s in %s", digest, repo) + } + + newPhase := func(tree *image.ImagesTree, cacheStorages ...storage.StagesStorage) *BuildPhase { + storageManager := &manager.StorageManager{ + StagesStorage: storage.NewRepoStagesStorage(&storage.NewRepoStagesStorageOptions{RepoAddress: stagesRepo}), + CacheStagesStorageList: cacheStorages, + } + return &BuildPhase{ + BasePhase: BasePhase{Conveyor: &Conveyor{imagesTree: tree, StorageManager: storageManager}}, + sbomStep: &sbomStep{}, + } + } + + newImage := func(ctx SpecContext, platform, repo, digest string, final *imagePkg.StageDesc) *image.Image { + img, err := image.NewImage(ctx, platform, "app", image.NoBaseImage, image.ImageOptions{}) + Expect(err).To(Succeed()) + + img.SetContentTagDesc(&imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: repo, RepoDigest: repo + "@" + digest}, + }) + if final != nil { + img.SetFinalContentTagDesc(final) + } + return img + } + + cacheStorage := func(address string) storage.StagesStorage { + s := mock.NewMockStagesStorage(gomock.NewController(GinkgoT())) + s.EXPECT().Address().Return(address).AnyTimes() + s.EXPECT().String().Return(address).AnyTimes() + return s + } + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + stagesRepo = host + "/test/stages" + finalRepo = host + "/test/final" + cacheRepo = host + "/test/cache" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + }) + + AfterEach(func() { + server.Close() + }) + + It("carries the artifacts of a single-platform image into the final repo and the cache repo", func(ctx SpecContext) { + digest := pushImage(ctx, stagesRepo, "v1") + attach(ctx, stagesRepo, digest, `{"scope":"app"}`) + copyManifestByDigest(ctx, stagesRepo, finalRepo, digest) + copyManifestByDigest(ctx, stagesRepo, cacheRepo, digest) + + finalDesc := &imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: finalRepo, RepoDigest: finalRepo + "@" + digest}, + } + + tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) + tree.AppendImageForTests(newImage(ctx, "linux/amd64", stagesRepo, digest, finalDesc)) + + Expect(newPhase(tree, cacheStorage(cacheRepo)).propagateArtifacts(ctx)).To(Succeed()) + + expectAttached(ctx, finalRepo, digest, `{"scope":"app"}`) + expectAttached(ctx, cacheRepo, digest, `{"scope":"app"}`) + }) + + It("leaves the artifacts alone when the image was not published to a final repo", func(ctx SpecContext) { + digest := pushImage(ctx, stagesRepo, "v1") + attach(ctx, stagesRepo, digest, `{"scope":"app"}`) + copyManifestByDigest(ctx, stagesRepo, finalRepo, digest) + + tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) + tree.AppendImageForTests(newImage(ctx, "linux/amd64", stagesRepo, digest, nil)) + + Expect(newPhase(tree).propagateArtifacts(ctx)).To(Succeed()) + + expectNothingAttached(ctx, finalRepo, digest) + }) + + It("carries per-platform artifacts onto the platform manifests and image-level artifacts onto the index", func(ctx SpecContext) { + platforms := []string{"linux/amd64", "linux/arm64"} + indexDigest, children := pushIndex(ctx, stagesRepo, "index", platforms) + + attach(ctx, stagesRepo, indexDigest, `{"scope":"image"}`) + attach(ctx, stagesRepo, children[0], `{"scope":"amd64"}`) + attach(ctx, stagesRepo, children[1], `{"scope":"arm64"}`) + + copyManifestByDigest(ctx, stagesRepo, finalRepo, indexDigest) + copyManifestByDigest(ctx, stagesRepo, cacheRepo, children[0]) + copyManifestByDigest(ctx, stagesRepo, cacheRepo, children[1]) + + tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) + images := make([]*image.Image, 0, len(platforms)) + for i, platform := range platforms { + img := newImage(ctx, platform, stagesRepo, children[i], nil) + images = append(images, img) + tree.AppendImageForTests(img) + } + + multiImg := image.NewMultiplatformImage("app", images, 0, 1) + multiImg.SetStageDesc(&imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: stagesRepo, RepoDigest: stagesRepo + "@" + indexDigest}, + }) + multiImg.SetFinalStageDesc(&imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: finalRepo, RepoDigest: finalRepo + "@" + indexDigest}, + }) + tree.SetMultiplatformImage(multiImg) + + Expect(newPhase(tree, cacheStorage(cacheRepo)).propagateArtifacts(ctx)).To(Succeed()) + + expectAttached(ctx, finalRepo, indexDigest, `{"scope":"image"}`) + expectAttached(ctx, finalRepo, children[0], `{"scope":"amd64"}`) + expectAttached(ctx, finalRepo, children[1], `{"scope":"arm64"}`) + expectAttached(ctx, cacheRepo, children[0], `{"scope":"amd64"}`) + expectAttached(ctx, cacheRepo, children[1], `{"scope":"arm64"}`) + expectNothingAttached(ctx, cacheRepo, indexDigest) + }) +}) diff --git a/pkg/build/build_phase_test.go b/pkg/build/build_phase_test.go index b7e89929ea..00a46fb9b9 100644 --- a/pkg/build/build_phase_test.go +++ b/pkg/build/build_phase_test.go @@ -388,12 +388,4 @@ var _ = Describe("BuildPhase", func() { Expect(digestDisabledAgain).To(Equal(digestBaseline)) }) }) - - Describe("finalStageDescForImage", func() { - It("returns nil for a single-platform image resolved from the cache, without a built stage image", func() { - phase := &BuildPhase{} - - Expect(phase.finalStageDescForImage("app", []*image.Image{{}})).To(BeNil()) - }) - }) }) diff --git a/pkg/build/build_report.go b/pkg/build/build_report.go index eb1b3a4717..791791338e 100644 --- a/pkg/build/build_report.go +++ b/pkg/build/build_report.go @@ -242,7 +242,7 @@ func createBuildReport(ctx context.Context, phase *BuildPhase, imagePairs []util targetPlatforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform }) for _, img := range images { - imageDesc := img.GetContentTagDesc() + imageDesc := img.GetPublishedContentTagDesc() var stages []ReportStageRecord if !img.AnchorReused { stages = getStagesReport(img, false) diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index c5f4db2339..24b6eb5fb7 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -539,7 +539,7 @@ func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) ([]*imag if len(platforms) == 1 { img := images[0] - getter := c.StorageManager.GetImageInfoGetter(img.Name, img.GetContentTagDesc(), opts) + getter := c.StorageManager.GetImageInfoGetter(img.Name, img.GetPublishedContentTagDesc(), opts) imagesGetters = append(imagesGetters, getter) } else { img := c.imagesTree.GetMultiplatformImage(name) @@ -581,7 +581,10 @@ func (c *Conveyor) GetImagesEnvArray() []string { continue } - envArray = append(envArray, GenerateImageEnv(img.Name, c.GetImageContentTagName(img.TargetPlatform, img.Name))) + // werf compose hands these references to docker compose, so they have to name + // the image where it was published — the final repo when the build used one, + // matching what GetImagesEnvArrayFromReport reads out of the build report. + envArray = append(envArray, GenerateImageEnv(img.Name, img.GetPublishedContentTagDesc().Info.Name)) } return envArray diff --git a/pkg/build/image/image.go b/pkg/build/image/image.go index c8402c06bc..fddcb8c4c6 100644 --- a/pkg/build/image/image.go +++ b/pkg/build/image/image.go @@ -138,8 +138,11 @@ type Image struct { stageDurations map[stage.StageName]time.Duration lastNonEmptyStage stage.Interface contentTagDesc *image.StageDesc - rebuilt bool - useCustomTag bool + // finalContentTagDesc is the content tag descriptor of the image as copied into + // the final repo; contentTagDesc keeps describing the image in the stages repo. + finalContentTagDesc *image.StageDesc + rebuilt bool + useCustomTag bool baseImageType BaseImageType baseImageReference string @@ -344,6 +347,26 @@ func (i *Image) GetContentTagDesc() *image.StageDesc { return i.contentTagDesc } +func (i *Image) SetFinalContentTagDesc(desc *image.StageDesc) { + i.finalContentTagDesc = desc +} + +func (i *Image) GetFinalContentTagDesc() *image.StageDesc { + return i.finalContentTagDesc +} + +// GetPublishedContentTagDesc returns the descriptor of the image as published for +// consumers: the final repo one when the image was copied there, the stages repo +// one otherwise. Build-internal consumers — artifact convergence, base and import +// SBOM lookups — must keep using GetContentTagDesc, which always describes the +// image in the repository it was built in. +func (i *Image) GetPublishedContentTagDesc() *image.StageDesc { + if i.finalContentTagDesc != nil { + return i.finalContentTagDesc + } + return i.contentTagDesc +} + func (i *Image) GetStage(name stage.StageName) stage.Interface { for _, s := range i.stages { if s.Name() == name { diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index 1946a0cd92..d6c84509d3 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -212,26 +212,35 @@ func (step *sbomStep) calculateStableChecksum(scanOpts scanner.ScanOptions, merg ) } -// PropagateArtifacts copies the artifacts attached to the image stage (e.g. its SBOM) -// into the final repo and the cache repos. Stages themselves are copied there before -// SBOM generation runs, so the artifacts have to catch up separately. -func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName string, stageDesc, finalStageDesc *image.StageDesc, cacheStagesStorageList []storage.StagesStorage) error { - srcRepo := stageDesc.Info.Repository - srcDigest := stageDesc.Info.GetDigest() - - if finalStageDesc != nil && finalStageDesc.Info.Repository != srcRepo { - if err := logboek.Context(ctx).Default().LogProcess("image %s: Copy SBOM artifacts into the final repo %s", werfImgName, finalStageDesc.Info.Repository).DoError(func() error { - return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, finalStageDesc.Info.Repository, finalStageDesc.Info.GetDigest()) +// PropagateArtifactsOptions carries the destinations of a propagation: the final repo +// descriptor of the same image when the build published one, and the cache stages +// storages the stage was placed in. +type PropagateArtifactsOptions struct { + FinalRepo string + FinalDigest string + CacheStagesStorageList []storage.StagesStorage +} + +// PropagateArtifacts copies the artifacts attached to the image in the repository it +// was built in — its SBOM, VEX and any other attached kind — into the final repo and +// the cache repos. Stages are copied there before the artifacts exist, so the +// artifacts have to catch up separately. The copy runs on every build and is +// idempotent, so a destination holding the image without its artifacts is repaired +// by the next run. +func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName, srcRepo, srcDigest string, opts PropagateArtifactsOptions) error { + if opts.FinalRepo != "" && opts.FinalRepo != srcRepo { + if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy attached artifacts into the final repo %s", werfImgName, opts.FinalRepo).DoError(func() error { + return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, opts.FinalRepo, opts.FinalDigest) }); err != nil { - return fmt.Errorf("copy attached artifacts into final repo %s: %w", finalStageDesc.Info.Repository, err) + return fmt.Errorf("copy attached artifacts into final repo %s: %w", opts.FinalRepo, err) } } - for _, cache := range cacheStagesStorageList { + for _, cache := range opts.CacheStagesStorageList { if cache.Address() == storage.LocalStorageAddress || cache.Address() == srcRepo { continue } - if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy SBOM artifacts into cache %s", werfImgName, cache.String()).DoError(func() error { + if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy attached artifacts into cache %s", werfImgName, cache.String()).DoError(func() error { return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, cache.Address(), srcDigest) }); err != nil { logboek.Context(ctx).Warn().LogF("Warning: unable to copy attached artifacts into cache stages storage %s: %s\n", cache.String(), err) diff --git a/pkg/build/sbom_step_propagate_test.go b/pkg/build/sbom_step_propagate_test.go index 0004e1cfef..3c4726b3f0 100644 --- a/pkg/build/sbom_step_propagate_test.go +++ b/pkg/build/sbom_step_propagate_test.go @@ -15,7 +15,6 @@ import ( "github.com/werf/werf/v3/pkg/attestation" "github.com/werf/werf/v3/pkg/docker_registry" - werfImage "github.com/werf/werf/v3/pkg/image" "github.com/werf/werf/v3/pkg/oci/artifact" "github.com/werf/werf/v3/pkg/storage" "github.com/werf/werf/v3/test/mock" @@ -55,16 +54,6 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) } - stageDescFor := func(repo, digest string) *werfImage.StageDesc { - return &werfImage.StageDesc{ - StageID: &werfImage.StageID{}, - Info: &werfImage.Info{ - Repository: repo, - RepoDigest: repo + "@" + digest, - }, - } - } - cacheStorage := func(address string) storage.StagesStorage { s := mock.NewMockStagesStorage(gomock.NewController(GinkgoT())) s.EXPECT().Address().Return(address).AnyTimes() @@ -96,7 +85,7 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { copyImageByDigest(ctx, srcRepo, finalRepo, srcDigest) step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), stageDescFor(finalRepo, srcDigest), nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{FinalRepo: finalRepo, FinalDigest: srcDigest})).To(Succeed()) finalStore := artifact.NewOCIStore(finalRepo, "app", remoteOpts...) content, err := finalStore.GetAttachedContent(ctx, srcDigest, attestation.DSSEMediaType, nil) @@ -113,7 +102,7 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { cacheStorage(srcRepo), cacheStorage(cacheRepo), } - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), nil, caches)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{CacheStagesStorageList: caches})).To(Succeed()) cacheStore := artifact.NewOCIStore(cacheRepo, "app", remoteOpts...) content, err := cacheStore.GetAttachedContent(ctx, srcDigest, attestation.DSSEMediaType, nil) @@ -123,23 +112,23 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { It("should do nothing without a final repo and caches", func(ctx SpecContext) { step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), nil, nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{})).To(Succeed()) }) It("should skip the final repo when it matches the stages repo", func(ctx SpecContext) { step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), stageDescFor(srcRepo, srcDigest), nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{FinalRepo: srcRepo, FinalDigest: srcDigest})).To(Succeed()) }) It("should not fail when a cache repo is unreachable", func(ctx SpecContext) { step := &sbomStep{} caches := []storage.StagesStorage{cacheStorage("127.0.0.1:1/unreachable/cache")} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), nil, caches)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{CacheStagesStorageList: caches})).To(Succeed()) }) It("should fail when the final repo copy fails", func(ctx SpecContext) { step := &sbomStep{} - err := step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), stageDescFor("127.0.0.1:1/unreachable/final", srcDigest), nil) + err := step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{FinalRepo: "127.0.0.1:1/unreachable/final", FinalDigest: srcDigest}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("copy attached artifacts into final repo")) }) diff --git a/pkg/build/stages/remote_storage.go b/pkg/build/stages/remote_storage.go index 9161beabb6..221d5cee19 100644 --- a/pkg/build/stages/remote_storage.go +++ b/pkg/build/stages/remote_storage.go @@ -8,6 +8,7 @@ import ( "github.com/werf/werf/v3/pkg/build" "github.com/werf/werf/v3/pkg/docker_registry" "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/oci/artifact" "github.com/werf/werf/v3/pkg/ref" "github.com/werf/werf/v3/pkg/storage/manager" ) @@ -83,6 +84,12 @@ func (s *RemoteStorage) copyCurrentBuildStagesFromRemote(ctx context.Context, fr if err = fromRemote.RegistryClient.CopyImage(ctx, infoGetterName, reference.FullName(), docker_registry.CopyImageOptions{}); err != nil { return fmt.Errorf("error copying stage %s into %s: %w", infoGetterName, reference.FullName(), err) } + + if infoGetter.Digest != "" { + if err := artifact.CopyAllAttachedArtifacts(ctx, infoGetter.Repo, infoGetter.Digest, reference.Repo, infoGetter.Digest); err != nil { + return fmt.Errorf("error copying artifacts attached to stage %s into %s: %w", infoGetterName, reference.Repo, err) + } + } } return nil @@ -116,6 +123,12 @@ func (s *RemoteStorage) copyAllFromRemote(ctx context.Context, fromRemote *Remot if err = fromRemote.RegistryClient.CopyImage(ctx, stageName, reference.FullName(), docker_registry.CopyImageOptions{}); err != nil { return fmt.Errorf("error copying stage %s into %s: %w", stageName, reference.FullName(), err) } + + if digest := stageDesc.Info.GetDigest(); digest != "" { + if err := artifact.CopyAllAttachedArtifacts(ctx, stageDesc.Info.Repository, digest, reference.Repo, digest); err != nil { + return fmt.Errorf("error copying artifacts attached to stage %s into %s: %w", stageName, reference.Repo, err) + } + } } return nil diff --git a/pkg/deploy/bundles/copy_artifacts_test.go b/pkg/deploy/bundles/copy_artifacts_test.go new file mode 100644 index 0000000000..904b9350ee --- /dev/null +++ b/pkg/deploy/bundles/copy_artifacts_test.go @@ -0,0 +1,145 @@ +package bundles + +import ( + "fmt" + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + chartcommon "github.com/werf/nelm/v2/pkg/helm/pkg/chart/common" + chart "github.com/werf/nelm/v2/pkg/helm/pkg/chart/v2" + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/logging" + "github.com/werf/werf/v3/pkg/oci/artifact" + bundles_registry "github.com/werf/werf/v3/pkg/ref" +) + +var _ = Describe("Bundle copy artifacts", func() { + const imageTag = "tag-1" + + var ( + server *httptest.Server + srcRepo string + dstRepo string + remoteOpts []remote.Option + ) + + pushImage := func(ctx SpecContext, repo string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + ref, err := name.NewTag(repo + ":" + imageTag) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + return dgst.String() + } + + copyImageByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + img, err := remote.Image(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + // copyBundle runs a remote-to-remote bundle copy of a single-image bundle. The + // bundle copy itself goes through the stub, which holds archives rather than + // manifests, so the destination manifest has to be placed separately — at the + // same digest, the way a registry copy places it. + copyBundle := func(ctx SpecContext, digest string) { + srcRef := fmt.Sprintf("%s:%s", srcRepo, imageTag) + ch := &chart.Chart{ + Metadata: &chart.Metadata{ + APIVersion: "v2", + Name: "testproject", + Version: "1.2.3", + Type: "application", + }, + Values: map[string]interface{}{ + "werf": map[string]interface{}{ + "image": map[string]interface{}{"image-1": srcRef}, + "repo": srcRepo, + }, + }, + Raw: []*chartcommon.File{ + { + Name: "values.yaml", + Data: []byte(fmt.Sprintf("werf:\n image:\n image-1: %s\n repo: %s\n", srcRef, srcRepo)), + }, + }, + } + + bundlesRegistryClient := NewBundlesRegistryClientStub() + registryClient := NewDockerRegistryStub() + registryClient.ImagesByReference[srcRef] = []byte(`image-1-bytes`) + registryClient.RepoImagesByReference[srcRef] = &image.Info{RepoDigest: srcRepo + "@" + digest} + + fromAddr, err := bundles_registry.ParseAddr(srcRepo + ":1.2.3") + Expect(err).NotTo(HaveOccurred()) + from := NewRemoteBundle(fromAddr.RegistryAddress, bundlesRegistryClient, registryClient) + bundlesRegistryClient.StubCharts[fromAddr.RegistryAddress.FullName()] = ch + + toAddr, err := bundles_registry.ParseAddr(dstRepo + ":4.5.6") + Expect(err).NotTo(HaveOccurred()) + to := NewRemoteBundle(toAddr.RegistryAddress, bundlesRegistryClient, registryClient) + + Expect(from.CopyTo(logging.WithLogger(ctx), to, copyToOptions{})).To(Succeed()) + } + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + srcRepo = host + "/group/testproject" + dstRepo = host + "/group2/testproject2" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + }) + + AfterEach(func() { + server.Close() + }) + + It("should carry the artifacts of the images it copies between registries", func(ctx SpecContext) { + digest := pushImage(ctx, srcRepo) + + srcStore := artifact.NewOCIStore(srcRepo, "image-1", remoteOpts...) + Expect(srcStore.Attach(ctx, digest, attestation.DSSEMediaType, []byte(`{"sbom":true}`), "checksum-v1", "", "")).To(Succeed()) + + copyImageByDigest(ctx, srcRepo, dstRepo, digest) + copyBundle(ctx, digest) + + dstStore := artifact.NewOCIStore(dstRepo, "image-1", remoteOpts...) + content, err := dstStore.GetAttachedContent(ctx, digest, attestation.DSSEMediaType, nil) + Expect(err).To(Succeed()) + Expect(content).To(MatchJSON(`{"sbom":true}`)) + }) + + It("should copy a bundle whose images carry no artifacts", func(ctx SpecContext) { + digest := pushImage(ctx, srcRepo) + + copyImageByDigest(ctx, srcRepo, dstRepo, digest) + copyBundle(ctx, digest) + + idx, err := artifact.PullFallbackIndex(ctx, dstRepo, digest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + Expect(im.Manifests).To(BeEmpty()) + }) +}) diff --git a/pkg/deploy/bundles/copy_test.go b/pkg/deploy/bundles/copy_test.go index 18dc000f57..8e2031269f 100644 --- a/pkg/deploy/bundles/copy_test.go +++ b/pkg/deploy/bundles/copy_test.go @@ -15,6 +15,7 @@ import ( chartcommon "github.com/werf/nelm/v2/pkg/helm/pkg/chart/common" chart "github.com/werf/nelm/v2/pkg/helm/pkg/chart/v2" "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" "github.com/werf/werf/v3/pkg/logging" bundles_registry "github.com/werf/werf/v3/pkg/ref" ) @@ -644,12 +645,14 @@ func (client *BundlesRegistryClientStub) PushChart(ctx context.Context, ref *bun type DockerRegistryStub struct { docker_registry.Interface - ImagesByReference map[string][]byte + ImagesByReference map[string][]byte + RepoImagesByReference map[string]*image.Info } func NewDockerRegistryStub() *DockerRegistryStub { return &DockerRegistryStub{ - ImagesByReference: make(map[string][]byte), + ImagesByReference: make(map[string][]byte), + RepoImagesByReference: make(map[string]*image.Info), } } @@ -686,6 +689,13 @@ func (registry *DockerRegistryStub) PullImageArchive(ctx context.Context, archiv return nil } +// TryGetRepoImage resolves only what a spec put into RepoImagesByReference. The +// stub holds archives rather than registry manifests, so by default an image has +// no digest to carry artifacts for and the artifact-carrying step is skipped. +func (registry *DockerRegistryStub) TryGetRepoImage(_ context.Context, reference string) (*image.Info, error) { + return registry.RepoImagesByReference[reference], nil +} + func (registry *DockerRegistryStub) CopyImage(_ context.Context, sourceReference, destinationReference string, _ docker_registry.CopyImageOptions) error { data, hasImage := registry.ImagesByReference[sourceReference] if !hasImage { diff --git a/pkg/deploy/bundles/remote_bundle.go b/pkg/deploy/bundles/remote_bundle.go index 085069f023..1f31b7ccd2 100644 --- a/pkg/deploy/bundles/remote_bundle.go +++ b/pkg/deploy/bundles/remote_bundle.go @@ -13,6 +13,7 @@ import ( nelmcommon "github.com/werf/nelm/v2/pkg/common" chart "github.com/werf/nelm/v2/pkg/helm/pkg/chart/v2" "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/oci/artifact" bundles_registry "github.com/werf/werf/v3/pkg/ref" ) @@ -182,6 +183,7 @@ func (bundle *RemoteBundle) CopyFromRemote(ctx context.Context, fromRemote *Remo return err } + srcRepo := ref.Repo ref.Repo = bundle.RegistryAddress.Repo // TODO: copy images in parallel @@ -192,6 +194,14 @@ func (bundle *RemoteBundle) CopyFromRemote(ctx context.Context, fromRemote *Remo if err := fromRemote.RegistryClient.CopyImage(ctx, image, ref.FullName(), docker_registry.CopyImageOptions{}); err != nil { return fmt.Errorf("error copying image %s into %s: %w", image, ref.FullName(), err) } + + if imgInfo, err := fromRemote.RegistryClient.TryGetRepoImage(ctx, image); err != nil { + return fmt.Errorf("error resolving image %s to copy its attached artifacts: %w", image, err) + } else if imgInfo != nil { + if err := artifact.CopyAllAttachedArtifacts(ctx, srcRepo, imgInfo.GetDigest(), ref.Repo, imgInfo.GetDigest()); err != nil { + return fmt.Errorf("error copying artifacts attached to image %s into %s: %w", image, ref.Repo, err) + } + } } newImageVals[imageName] = ref.FullName() diff --git a/pkg/oci/artifact/copy.go b/pkg/oci/artifact/copy.go index 70a2da9ffb..161d2b64bd 100644 --- a/pkg/oci/artifact/copy.go +++ b/pkg/oci/artifact/copy.go @@ -2,16 +2,54 @@ package artifact import ( "context" + "errors" "fmt" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/werf/logboek" "github.com/werf/werf/v3/pkg/docker_registry" "github.com/werf/werf/v3/pkg/image" ) +// CopyAllAttachedArtifacts copies every artifact attached to srcDigest in srcRepo +// onto dstDigest in dstRepo, and when srcDigest is an image index it also copies +// the artifacts attached to every manifest the index references. Referenced +// manifests are addressed by the same digest in both repositories, which holds for +// registry-level copies — the only in-scope way an index travels between +// repositories. +func CopyAllAttachedArtifacts(ctx context.Context, srcRepo, srcDigest, dstRepo, dstDigest string, opts ...remote.Option) error { + if err := CopyAttachedArtifacts(ctx, srcRepo, srcDigest, dstRepo, dstDigest, opts...); err != nil { + return err + } + + entries, err := ListIndexPlatforms(ctx, srcRepo, srcDigest, opts...) + if err != nil { + // A source that does not hold the manifest holds no artifacts attached to it + // either, which is the same no-op CopyAttachedArtifacts makes of a missing + // fallback index rather than a reason to fail the operation that copied the + // image. + var transportErr *transport.Error + if errors.As(err, &transportErr) && transportErr.StatusCode == 404 { + return nil + } + return fmt.Errorf("list index manifests of %s: %w", srcRepo+"@"+srcDigest, err) + } + + for _, entry := range entries { + if entry.Digest == srcDigest { + continue + } + if err := CopyAttachedArtifacts(ctx, srcRepo, entry.Digest, dstRepo, entry.Digest, opts...); err != nil { + return fmt.Errorf("copy artifacts of index manifest %s: %w", entry.Digest, err) + } + } + + return nil +} + // CopyAttachedArtifacts copies every artifact attached to srcDigest in srcRepo onto // dstDigest in dstRepo. Artifacts are re-attached from their payload rather than // copied manifest-by-manifest, so the parent digests may differ (e.g. when a stage diff --git a/pkg/oci/artifact/copy_test.go b/pkg/oci/artifact/copy_test.go index 1470a053aa..85ff0edfac 100644 --- a/pkg/oci/artifact/copy_test.go +++ b/pkg/oci/artifact/copy_test.go @@ -1,6 +1,9 @@ package artifact_test import ( + "fmt" + "io" + "net/http" "net/http/httptest" "strings" @@ -8,6 +11,8 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/remote" . "github.com/onsi/ginkgo/v2" @@ -149,4 +154,132 @@ var _ = Describe("CopyAttachedArtifacts (integration)", func() { im := pullIndex(ctx, dstRepo, srcDigest) Expect(im.Manifests).To(BeEmpty()) }) + + Describe("CopyAllAttachedArtifacts", func() { + // The index carries a platform per manifest the way a werf multi-platform + // image does: entries without one are not platform manifests and are not + // traversed. + pushMultiplatformIndex := func(ctx SpecContext, repo, tag string) (string, []string) { + idx := v1.ImageIndex(empty.Index) + var children []string + + for _, platform := range []string{"linux/amd64", "linux/arm64"} { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + parsed, err := v1.ParsePlatform(platform) + Expect(err).To(Succeed()) + + idx = mutate.AppendManifests(idx, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{Platform: parsed}, + }) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + children = append(children, dgst.String()) + } + + ref, err := name.NewTag(repo + ":" + tag) + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(ref, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := idx.Digest() + Expect(err).To(Succeed()) + + return dgst.String(), children + } + + copyIndexByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + idx, err := remote.Index(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(toRef, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + It("should carry the artifacts of every manifest an index references", func(ctx SpecContext) { + indexDigest, children := pushMultiplatformIndex(ctx, srcRepo, "index") + + store := artifact.NewOCIStore(srcRepo, "my-app", remoteOpts...) + Expect(store.Attach(ctx, indexDigest, artifactType, []byte(`{"scope":"image"}`), "checksum-index", "", "")).To(Succeed()) + for i, child := range children { + payload := fmt.Sprintf(`{"scope":"platform-%d"}`, i) + Expect(store.Attach(ctx, child, artifactType, []byte(payload), fmt.Sprintf("checksum-%d", i), "", "")).To(Succeed()) + } + + copyIndexByDigest(ctx, srcRepo, dstRepo, indexDigest) + + Expect(artifact.CopyAllAttachedArtifacts(ctx, srcRepo, indexDigest, dstRepo, indexDigest, remoteOpts...)).To(Succeed()) + + dstStore := artifact.NewOCIStore(dstRepo, "my-app", remoteOpts...) + content, err := dstStore.GetAttachedContent(ctx, indexDigest, artifactType, nil) + Expect(err).To(Succeed()) + Expect(content).To(MatchJSON(`{"scope":"image"}`)) + + for i, child := range children { + childContent, err := dstStore.GetAttachedContent(ctx, child, artifactType, nil) + Expect(err).To(Succeed()) + Expect(childContent).To(MatchJSON(fmt.Sprintf(`{"scope":"platform-%d"}`, i))) + } + }) + + It("should not move an artifact of a referenced manifest onto the index digest", func(ctx SpecContext) { + indexDigest, children := pushMultiplatformIndex(ctx, srcRepo, "index") + + store := artifact.NewOCIStore(srcRepo, "my-app", remoteOpts...) + Expect(store.Attach(ctx, children[0], artifactType, []byte(`{"scope":"platform-0"}`), "checksum-0", "", "")).To(Succeed()) + + copyIndexByDigest(ctx, srcRepo, dstRepo, indexDigest) + + Expect(artifact.CopyAllAttachedArtifacts(ctx, srcRepo, indexDigest, dstRepo, indexDigest, remoteOpts...)).To(Succeed()) + + Expect(pullIndex(ctx, dstRepo, indexDigest).Manifests).To(BeEmpty()) + Expect(pullIndex(ctx, dstRepo, children[0]).Manifests).To(HaveLen(1)) + }) + + It("should be a no-op when the source does not hold the manifest", func(ctx SpecContext) { + absentDigest := pushRandomImage(ctx, dstRepo, "only-in-dst") + + Expect(artifact.CopyAllAttachedArtifacts(ctx, srcRepo, absentDigest, dstRepo, absentDigest, remoteOpts...)).To(Succeed()) + + Expect(pullIndex(ctx, dstRepo, absentDigest).Manifests).To(BeEmpty()) + }) + + It("should fail when the source manifest cannot be read for a reason other than absence", func(ctx SpecContext) { + indexDigest, _ := pushMultiplatformIndex(ctx, srcRepo, "index") + copyIndexByDigest(ctx, srcRepo, dstRepo, indexDigest) + + upstream := server.Listener.Addr().String() + failing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/manifests/"+indexDigest) { + http.Error(w, "registry unavailable", http.StatusInternalServerError) + return + } + r.URL.Scheme = "http" + r.URL.Host = upstream + r.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + for k, vs := range resp.Header { + w.Header()[k] = vs + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + defer failing.Close() + + failingSrcRepo := strings.TrimPrefix(failing.URL, "http://") + "/test/src" + + err := artifact.CopyAllAttachedArtifacts(ctx, failingSrcRepo, indexDigest, dstRepo, indexDigest, remoteOpts...) + Expect(err).To(MatchError(ContainSubstring("list index manifests"))) + }) + }) }) diff --git a/pkg/oci/artifact/fallback.go b/pkg/oci/artifact/fallback.go index 77e586bc19..11f0cc3b2c 100644 --- a/pkg/oci/artifact/fallback.go +++ b/pkg/oci/artifact/fallback.go @@ -19,6 +19,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/samber/lo" "github.com/werf/logboek" "github.com/werf/werf/v3/pkg/image" @@ -304,6 +305,37 @@ func GetAttached(ctx context.Context, repo, parentDigest, artifactType, imageNam return matches[0], true, nil } +// ListUnregenerableArtifacts returns the predicate types of the artifacts attached to +// parentDigest that a build cannot produce again. Everything werf generates itself — +// the SBOM and the VEX document — carries the checksum annotation of its cache +// identity, while an attestation signed through werf attest sign carries none: its +// predicate and its signing key exist only on the user's side. +func ListUnregenerableArtifacts(ctx context.Context, repo, parentDigest string, opts ...remote.Option) ([]string, error) { + idx, err := pullFallbackIndex(ctx, repo, parentDigest, opts...) + if err != nil { + return nil, err + } + + im, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("read fallback index manifest: %w", err) + } + + var predicateTypes []string + for _, desc := range im.Manifests { + if desc.ArtifactType == "" || desc.Annotations[image.WerfChecksumAnnotation] != "" { + continue + } + predicateType := desc.Annotations[PredicateTypeAnnotation] + if predicateType == "" { + predicateType = desc.ArtifactType + } + predicateTypes = append(predicateTypes, predicateType) + } + + return lo.Uniq(predicateTypes), nil +} + func multipleArtifactEntriesWarning(parentDigest string, matches []v1.Descriptor) string { names := make([]string, 0, len(matches)) for _, desc := range matches { diff --git a/pkg/oci/artifact/unregenerable_test.go b/pkg/oci/artifact/unregenerable_test.go new file mode 100644 index 0000000000..fab689e72a --- /dev/null +++ b/pkg/oci/artifact/unregenerable_test.go @@ -0,0 +1,73 @@ +package artifact_test + +import ( + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/oci/artifact" +) + +var _ = Describe("ListUnregenerableArtifacts (integration)", func() { + var ( + server *httptest.Server + repo string + parentDigest string + remoteOpts []remote.Option + ) + + BeforeEach(func(ctx SpecContext) { + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + repo = host + "/test/app" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + + parent, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + parentRef, err := name.NewTag(repo + ":v1") + Expect(err).To(Succeed()) + Expect(remote.Write(parentRef, parent, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := parent.Digest() + Expect(err).To(Succeed()) + parentDigest = dgst.String() + }) + + AfterEach(func() { + server.Close() + }) + + It("should report nothing for a digest without artifacts", func(ctx SpecContext) { + Expect(artifact.ListUnregenerableArtifacts(ctx, repo, parentDigest, remoteOpts...)).To(BeEmpty()) + }) + + It("should not report artifacts a build regenerates", func(ctx SpecContext) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + Expect(store.Attach(ctx, parentDigest, attestation.DSSEMediaType, []byte(`{"sbom":true}`), "checksum-sbom", "linux/amd64", "https://cyclonedx.org/bom")).To(Succeed()) + + Expect(artifact.ListUnregenerableArtifacts(ctx, repo, parentDigest, remoteOpts...)).To(BeEmpty()) + }) + + It("should report an attestation signed outside the build", func(ctx SpecContext) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + Expect(store.Attach(ctx, parentDigest, attestation.DSSEMediaType, []byte(`{"sbom":true}`), "checksum-sbom", "", "https://cyclonedx.org/bom")).To(Succeed()) + Expect(store.Attach(ctx, parentDigest, attestation.DSSEMediaType, []byte(`{"custom":true}`), "", "", "https://example.com/predicate/v1")).To(Succeed()) + + idx, err := artifact.PullFallbackIndex(ctx, repo, parentDigest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + Expect(im.Manifests).To(HaveLen(2), "both artifacts must coexist for this spec to discriminate") + + Expect(artifact.ListUnregenerableArtifacts(ctx, repo, parentDigest, remoteOpts...)).To(ConsistOf("https://example.com/predicate/v1")) + }) +}) diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index d537e1dd51..978fa1738b 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -825,8 +825,22 @@ func (m *StorageManager) CopySuitableStageDescByDigest(ctx context.Context, stag return nil, fmt.Errorf("unable to get stage %s description from %s: %w", stageDesc.StageID.String(), destinationStagesStorage.String(), err) } else { if sourceStagesStorage.Address() != storage.LocalStorageAddress && destinationStagesStorage.Address() != storage.LocalStorageAddress { - if err := artifact.CopyAttachedArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest(), destinationStagesStorage.Address(), destinationStageDesc.Info.GetDigest()); err != nil { - return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageDesc.StageID.String(), err) + // The backend-mediated copy does not guarantee digest preservation. Artifacts + // describe the source digest; when it changed, werf regenerates its own by + // convergence and only what it cannot regenerate is worth reporting. + if destinationStageDesc.Info.GetDigest() == stageDesc.Info.GetDigest() { + if err := artifact.CopyAllAttachedArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest(), destinationStagesStorage.Address(), destinationStageDesc.Info.GetDigest()); err != nil { + return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageDesc.StageID.String(), err) + } + } else { + leftBehind, err := artifact.ListUnregenerableArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest()) + if err != nil { + return nil, fmt.Errorf("unable to list artifacts attached to stage %s in %s: %w", stageDesc.StageID.String(), sourceStagesStorage.String(), err) + } + if len(leftBehind) > 0 { + logboek.Context(ctx).Warn().LogF("WARNING: attestations [%s] of stage %s stay in %s: the copy changed the image digest, re-issue them against %s\n", + strings.Join(leftBehind, ", "), stageDesc.StageID.String(), sourceStagesStorage.String(), destinationStageDesc.Info.GetDigest()) + } } } return destinationStageDesc, nil diff --git a/pkg/storage/manager/storage_manager_copy_suitable_test.go b/pkg/storage/manager/storage_manager_copy_suitable_test.go new file mode 100644 index 0000000000..abc0f229fe --- /dev/null +++ b/pkg/storage/manager/storage_manager_copy_suitable_test.go @@ -0,0 +1,172 @@ +package manager + +import ( + "context" + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/container_backend" + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/logging" + "github.com/werf/werf/v3/pkg/oci/artifact" + "github.com/werf/werf/v3/pkg/storage" + "github.com/werf/werf/v3/pkg/werf" + "github.com/werf/werf/v3/test/mock" +) + +// copySuitableFakeStorage stands in for a registry-backed stages storage on the +// backend-mediated copy: it answers with the descriptor the stage has in its +// repository and accepts the fetch and the store without touching a backend. +type copySuitableFakeStorage struct { + storage.PrimaryStagesStorage + + address string + desc *image.StageDesc +} + +func (f *copySuitableFakeStorage) Address() string { return f.address } +func (f *copySuitableFakeStorage) String() string { return f.address } + +func (f *copySuitableFakeStorage) ConstructStageImageName(_, digest string, creationTs int64) string { + return f.address + ":" + image.NewStageID(digest, creationTs).String() +} + +func (f *copySuitableFakeStorage) FetchImage(_ context.Context, _ container_backend.LegacyImageInterface) error { + return nil +} + +func (f *copySuitableFakeStorage) StoreImage(_ context.Context, _ container_backend.LegacyImageInterface) error { + return nil +} + +func (f *copySuitableFakeStorage) GetStageDesc(_ context.Context, _ string, _ image.StageID) (*image.StageDesc, error) { + return f.desc, nil +} + +var _ = Describe("StorageManager.CopySuitableStageDescByDigest", func() { + var ( + server *httptest.Server + srcRepo string + dstRepo string + remoteOpts []remote.Option + stageID *image.StageID + ) + + pushRandomImage := func(ctx SpecContext, repo string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + ref, err := name.NewTag(repo + ":v1") + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + return dgst.String() + } + + copyImageByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + img, err := remote.Image(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + stageDescIn := func(repo, digest string) *image.StageDesc { + return &image.StageDesc{ + StageID: stageID, + Info: &image.Info{Name: repo + ":" + stageID.String(), Repository: repo, RepoDigest: repo + "@" + digest}, + } + } + + newManager := func(dst *copySuitableFakeStorage) (*StorageManager, container_backend.ContainerBackend) { + backend := mock.NewMockContainerBackend(gomock.NewController(GinkgoT())) + backend.EXPECT().RenameImage(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + return &StorageManager{ProjectName: "project", StagesStorage: dst}, backend + } + + attachedPayloads := func(ctx SpecContext, repo, digest string) []string { + idx, err := artifact.PullFallbackIndex(ctx, repo, digest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + + var payloads []string + for _, desc := range im.Manifests { + payloads = append(payloads, desc.Annotations[artifact.PredicateTypeAnnotation]+"|"+desc.Annotations[image.WerfChecksumAnnotation]) + } + return payloads + } + + BeforeEach(func(ctx SpecContext) { + Expect(werf.Init(GinkgoT().TempDir(), GinkgoT().TempDir())).To(Succeed()) + Expect(image.Init()).To(Succeed()) + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + srcRepo = host + "/test/secondary" + dstRepo = host + "/test/primary" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + stageID = image.NewStageID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1) + }) + + AfterEach(func() { + server.Close() + }) + + It("carries the attached artifacts when the copy preserved the digest", func(ctx SpecContext) { + digest := pushRandomImage(ctx, srcRepo) + copyImageByDigest(ctx, srcRepo, dstRepo, digest) + + srcStore := artifact.NewOCIStore(srcRepo, "app", remoteOpts...) + Expect(srcStore.Attach(ctx, digest, attestation.DSSEMediaType, []byte(`{"sbom":1}`), "checksum-sbom", "", "")).To(Succeed()) + + src := ©SuitableFakeStorage{address: srcRepo} + dst := ©SuitableFakeStorage{address: dstRepo, desc: stageDescIn(dstRepo, digest)} + m, backend := newManager(dst) + + desc, err := m.CopySuitableStageDescByDigest(logging.WithLogger(ctx), stageDescIn(srcRepo, digest), src, dst, backend, "linux/amd64") + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(digest)) + + Expect(attachedPayloads(ctx, dstRepo, digest)).To(HaveLen(1)) + }) + + It("carries nothing when the copy changed the digest", func(ctx SpecContext) { + srcDigest := pushRandomImage(ctx, srcRepo) + dstDigest := pushRandomImage(ctx, dstRepo) + Expect(dstDigest).NotTo(Equal(srcDigest)) + + srcStore := artifact.NewOCIStore(srcRepo, "app", remoteOpts...) + Expect(srcStore.Attach(ctx, srcDigest, attestation.DSSEMediaType, []byte(`{"sbom":1}`), "checksum-sbom", "", "")).To(Succeed()) + Expect(srcStore.Attach(ctx, srcDigest, attestation.DSSEMediaType, []byte(`{"signed":1}`), "", "", "https://example.com/user-predicate")).To(Succeed()) + + src := ©SuitableFakeStorage{address: srcRepo} + dst := ©SuitableFakeStorage{address: dstRepo, desc: stageDescIn(dstRepo, dstDigest)} + m, backend := newManager(dst) + + desc, err := m.CopySuitableStageDescByDigest(logging.WithLogger(ctx), stageDescIn(srcRepo, srcDigest), src, dst, backend, "linux/amd64") + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(dstDigest)) + + Expect(attachedPayloads(ctx, dstRepo, dstDigest)).To(BeEmpty(), "an artifact about %s must not be attached to %s", srcDigest, dstDigest) + Expect(attachedPayloads(ctx, dstRepo, srcDigest)).To(BeEmpty()) + Expect(attachedPayloads(ctx, srcRepo, srcDigest)).To(HaveLen(2), "the source keeps what was left behind") + }) +}) diff --git a/pkg/storage/meta_repo_marker_test.go b/pkg/storage/meta_repo_marker_test.go index 7331168bd3..dde533c291 100644 --- a/pkg/storage/meta_repo_marker_test.go +++ b/pkg/storage/meta_repo_marker_test.go @@ -167,7 +167,11 @@ func (r *markerRegistry) CopyImage(_ context.Context, sourceReference, destinati for k, v := range src.Labels { labels[k] = v } - r.images[destinationReference] = &image.Info{Name: destinationReference, Tag: refTag(destinationReference), Labels: labels} + dst := &image.Info{Name: destinationReference, Tag: refTag(destinationReference), Labels: labels} + if src.RepoDigest != "" { + dst.RepoDigest = refRepo(destinationReference) + "@" + strings.SplitN(src.RepoDigest, "@", 2)[1] + } + r.images[destinationReference] = dst return nil } diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 1b2f1d33dc..286748f132 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -946,6 +946,12 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } if desc != nil { + // The manifest may be in place without its artifacts; the copy is idempotent + // and repairs that. The digest is the same on both sides because a stage + // reaches this destination through a registry-level copy. + if err := artifact.CopyAllAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { + return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageID, err) + } return desc, nil } @@ -960,7 +966,7 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } - if err := artifact.CopyAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { + if err := artifact.CopyAllAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageID, err) } diff --git a/pkg/storage/repo_stages_storage_copy_test.go b/pkg/storage/repo_stages_storage_copy_test.go new file mode 100644 index 0000000000..63386134ca --- /dev/null +++ b/pkg/storage/repo_stages_storage_copy_test.go @@ -0,0 +1,133 @@ +package storage + +import ( + "context" + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/oci/artifact" +) + +type copyFromStorageRegistryStub struct { + docker_registry.Interface + + repoDigestByReference map[string]string + copied []string +} + +func (r *copyFromStorageRegistryStub) GetRepoImage(_ context.Context, reference string) (*image.Info, error) { + repoDigest, found := r.repoDigestByReference[reference] + if !found { + return nil, &manifestUnknownError{} + } + return &image.Info{Name: reference, RepoDigest: repoDigest}, nil +} + +func (r *copyFromStorageRegistryStub) IsTagExist(_ context.Context, _ string, _ ...docker_registry.Option) (bool, error) { + return false, nil +} + +func (r *copyFromStorageRegistryStub) CopyImage(_ context.Context, sourceReference, destinationReference string, _ docker_registry.CopyImageOptions) error { + r.copied = append(r.copied, sourceReference+" -> "+destinationReference) + return nil +} + +type manifestUnknownError struct{} + +func (e *manifestUnknownError) Error() string { return "MANIFEST_UNKNOWN: manifest unknown" } + +var _ = Describe("RepoStagesStorage.CopyFromStorage", func() { + const projectName = "project" + + var ( + server *httptest.Server + srcRepo string + dstRepo string + remoteOpts []remote.Option + stageID image.StageID + ) + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + srcRepo = host + "/test/src" + dstRepo = host + "/test/dst" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + stageID = *image.NewStageID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1) + }) + + AfterEach(func() { + server.Close() + }) + + pushStage := func(ctx SpecContext, repos ...string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + dgst, err := img.Digest() + Expect(err).To(Succeed()) + + for _, repo := range repos { + ref, err := name.NewDigest(repo + "@" + dgst.String()) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + return dgst.String() + } + + newStorage := func(repo string, stub *copyFromStorageRegistryStub) *RepoStagesStorage { + return NewRepoStagesStorage(&NewRepoStagesStorageOptions{ + RepoAddress: repo, + DockerRegistry: stub, + SkipMetaCheck: true, + }) + } + + It("attaches the artifacts of the source to a destination that already holds the stage without them", func(ctx SpecContext) { + digest := pushStage(ctx, srcRepo, dstRepo) + srcStore := artifact.NewOCIStore(srcRepo, "app", remoteOpts...) + Expect(srcStore.Attach(ctx, digest, attestation.DSSEMediaType, []byte(`{"v":1}`), "checksum-v1", "", "")).To(Succeed()) + + src := newStorage(srcRepo, ©FromStorageRegistryStub{}) + stub := ©FromStorageRegistryStub{repoDigestByReference: map[string]string{ + dstRepo + ":" + stageID.String(): dstRepo + "@" + digest, + }} + dst := newStorage(dstRepo, stub) + + desc, err := dst.CopyFromStorage(ctx, src, projectName, stageID, CopyFromStorageOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(digest)) + Expect(stub.copied).To(BeEmpty(), "the manifest was in place, nothing to copy") + + dstStore := artifact.NewOCIStore(dstRepo, "app", remoteOpts...) + content, err := dstStore.GetAttachedContent(ctx, digest, attestation.DSSEMediaType, nil) + Expect(err).To(Succeed()) + Expect(content).To(MatchJSON(`{"v":1}`)) + }) + + It("does not fail on a destination that already holds a stage the source does not", func(ctx SpecContext) { + digest := pushStage(ctx, dstRepo) + + src := newStorage(srcRepo, ©FromStorageRegistryStub{}) + dst := newStorage(dstRepo, ©FromStorageRegistryStub{repoDigestByReference: map[string]string{ + dstRepo + ":" + stageID.String(): dstRepo + "@" + digest, + }}) + + desc, err := dst.CopyFromStorage(ctx, src, projectName, stageID, CopyFromStorageOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(digest)) + }) +}) diff --git a/pkg/storage/stage_lookup_test.go b/pkg/storage/stage_lookup_test.go index 15b81b76d0..5f4b5f4210 100644 --- a/pkg/storage/stage_lookup_test.go +++ b/pkg/storage/stage_lookup_test.go @@ -13,6 +13,11 @@ import ( "github.com/werf/werf/v3/pkg/oci/artifact" ) +// stageLookupDigest is a syntactically valid digest for stages the stub registry +// fabricates: the artifact copy addresses the stage by digest and rejects an +// empty one before reaching the registry. +const stageLookupDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + var _ = ginkgo.Describe("stage lookup", func() { ginkgo.DescribeTable("returns an unavailable error instead of a nil descriptor", func(ctx ginkgo.SpecContext, present, rejected bool, expected error) { @@ -57,8 +62,8 @@ var _ = ginkgo.Describe("stage lookup", func() { stageID := image.NewStageID("digest", 1) sourceRef := source.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) destinationRef := destination.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) - registry.put(sourceRef, nil) - registry.put(destinationRef, nil) + putWithDigest(registry.markerRegistry, sourceRef, source.RepoAddress+"@"+stageLookupDigest) + putWithDigest(registry.markerRegistry, destinationRef, destination.RepoAddress+"@"+stageLookupDigest) registry.brokenImage = registry.images[destinationRef] desc, err := destination.CopyFromStorage(ctx, source, "project", *stageID, CopyFromStorageOptions{}) @@ -78,9 +83,9 @@ var _ = ginkgo.Describe("stage lookup", func() { stageID := image.NewStageID("digest", 1) sourceRef := source.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) destinationRef := destination.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) - registry.put(sourceRef, nil) + putWithDigest(registry.markerRegistry, sourceRef, source.RepoAddress+"@"+stageLookupDigest) if existing { - registry.put(destinationRef, nil) + putWithDigest(registry.markerRegistry, destinationRef, destination.RepoAddress+"@"+stageLookupDigest) } if rejected { registry.put(makeRepoRejectedStageImageRecord(destination.RepoAddress, stageID.Digest, stageID.CreationTs), nil) diff --git a/test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml new file mode 100644 index 0000000000..f5189f97b9 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml @@ -0,0 +1,18 @@ +project: werf-test-e2e-sbom-final-repo-dependent +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: base +from: registry.werf.io/werf/scratch:latest +git: + - add: / + to: /b +--- +image: app +fromImage: base +git: + - add: / + to: /a diff --git a/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml new file mode 100644 index 0000000000..4b2d999412 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml @@ -0,0 +1,23 @@ +project: werf-test-e2e-sbom-final-repo-import +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: carrier +from: registry.werf.io/werf/scratch:latest +git: + - add: / + to: /c +--- +image: app +from: registry.werf.io/werf/scratch:latest +git: + - add: / + to: /a +import: + - image: carrier + add: /c/werf.yaml + to: /imported/werf.yaml + before: setup diff --git a/test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml new file mode 100644 index 0000000000..4145936be1 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml @@ -0,0 +1,15 @@ +project: werf-test-e2e-sbom-final-repo-multiplatform +configVersion: 1 +build: + platform: + - linux/amd64 + - linux/arm64 + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: app +from: registry.werf.io/werf/scratch:latest +git: + - add: / + to: /app diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock new file mode 100644 index 0000000000..e65f33706f --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "cargo-simple" +version = "0.1.0" +dependencies = [ + "anyhow", +] diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml new file mode 100644 index 0000000000..8e21b0a428 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "cargo-simple" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.86" diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base b/test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base new file mode 100644 index 0000000000..71b5783d93 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base @@ -0,0 +1,4 @@ +FROM rust:1.78-slim +COPY --from=golang:1.22-bookworm /usr/local/go /usr/local/go +ENV PATH=/usr/local/go/bin:$PATH +LABEL io.deckhouse.internal.builder=true diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod new file mode 100644 index 0000000000..a51410178d --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod @@ -0,0 +1,5 @@ +module example.com/sweep-base + +go 1.21 + +require golang.org/x/text v0.14.0 diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum new file mode 100644 index 0000000000..c9c7c64d2d --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum @@ -0,0 +1,2 @@ +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs b/test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs new file mode 100644 index 0000000000..0b7841726c --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("sbom-final-repo-sweep"); +} diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json b/test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json new file mode 100644 index 0000000000..1d629a538f --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json @@ -0,0 +1,15 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://example.com/vex/e2e-test-001", + "author": "e2e-test", + "timestamp": "2024-06-01T00:00:00Z", + "statements": [ + { + "vulnerability": {"name": "CVE-2024-E2E001"}, + "products": [{"@id": "pkg:oci/werf-test-app"}], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "The vulnerable function is never called in this build." + } + ] +} \ No newline at end of file diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml b/test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml new file mode 100644 index 0000000000..9483c3670c --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml @@ -0,0 +1,5 @@ +giterminismConfigVersion: 1 +config: + goTemplateRendering: + allowEnvVariables: + - BUILDER_BASE_IMAGE diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml new file mode 100644 index 0000000000..14fd1e2fb3 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml @@ -0,0 +1,44 @@ +project: werf-test-e2e-sbom-final-repo-sweep +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: carrier +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: / + to: /c + stageDependencies: + packages: + - Cargo.toml + - Cargo.lock +packages: + - type: rust-cargo + workdir: /c +--- +image: base +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: /base + to: /b + stageDependencies: + packages: + - go.mod + - go.sum +packages: + - type: go-mod + workdir: /b +--- +image: app +fromImage: base +git: + - add: / + to: /a +import: + - image: carrier + add: /c/werf.yaml + to: /imported/werf.yaml + before: setup +vex: vex.openvex.json diff --git a/test/e2e/sbom/final_repo_cleanup_test.go b/test/e2e/sbom/final_repo_cleanup_test.go new file mode 100644 index 0000000000..340b2651ae --- /dev/null +++ b/test/e2e/sbom/final_repo_cleanup_test.go @@ -0,0 +1,117 @@ +package e2e_build_test + +import ( + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/crane" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/utils" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM retention across cleanup", Label("e2e", "sbom", "final-repo", "cleanup"), func() { + It("cleanup keeps the SBOM of a retained image in both repos and collects it once the image is deleted", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_cleanup" + SuiteData.InitTestRepo(ctx, repoDirname, "inject/ospm_basic") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-cleanup-builder") + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_cleanup.json"), + &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, + ) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo), + "expected build report to reference the final repo") + finalDigest := appRecord.DockerImageDigest + Expect(finalDigest).NotTo(BeEmpty()) + + stageTag := stageTagOf(ctx, werfProject, "app", builderEnv) + + assertSbomReadable := func(repo string, extraArgs ...string) { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: append([]string{"--repo", repo}, extraArgs...), + Envs: builderEnv, + }, + }) + sbomtest.AssertHasComponent(sbomtest.MustParseSBOMOutput(sbomOut), "curl", "8.12.1") + } + + By("reading the SBOM from both repos before cleanup") + assertSbomReadable(finalRepo, "--digest", finalDigest) + assertSbomReadable(stagesRepo, "--tag", stageTag) + + // The keep list pins the stage without relying on git history reachability, + // so the retention this spec asserts cannot pass because nothing was + // eligible for deletion in the first place: the same run deletes every stage + // that is not on the list. + keepListPath := filepath.Join(testRepoPath, ".werf-keep-list") + Expect(os.WriteFile(keepListPath, []byte(stageTag+"\n"), 0o600)).To(Succeed()) + + // werf cleanup requires a git remote origin; it lives outside the project + // directory so that it does not turn into an untracked file of the build. + bareRemotePath := filepath.Join(SuiteData.TmpDir, "sbom_final_repo_cleanup_remote.git") + utils.RunSucceedCommand(ctx, testRepoPath, "git", "init", "--bare", bareRemotePath) + utils.RunSucceedCommand(ctx, testRepoPath, "git", "remote", "add", "origin", bareRemotePath) + + cleanupArgs := []string{"cleanup", "--without-kube", "--keep-stages-built-within-last-n-hours=0"} + + By("running cleanup twice while the stage is on the keep list") + for range 2 { + werfProject.RunCommand(ctx, append(append([]string{}, cleanupArgs...), "--keep-list", keepListPath), + werf.CommonOptions{Envs: builderEnv}) + + assertSbomReadable(finalRepo, "--digest", finalDigest) + assertSbomReadable(stagesRepo, "--tag", stageTag) + } + + By("running cleanup with nothing protecting the stage") + werfProject.RunCommand(ctx, cleanupArgs, werf.CommonOptions{Envs: builderEnv}) + + registryOptions := []crane.Option{crane.Insecure, crane.WithContext(ctx)} + for _, repo := range []string{stagesRepo, finalRepo} { + tags, err := crane.ListTags(repo, registryOptions...) + Expect(err).NotTo(HaveOccurred()) + Expect(tags).NotTo(ContainElement(stageTag), "cleanup must have deleted the stage from %s", repo) + } + + By("checking the orphaned SBOM artifacts were collected in both repos") + expectNoSbomArtifact(ctx, stagesRepo, finalDigest) + expectNoSbomArtifact(ctx, finalRepo, finalDigest) + }) +}) + +// stageTagOf returns the content-based tag of the image's last stage in the stages repo. +// werf stage image prints the reference as its last line, so the preceding log lines are dropped. +func stageTagOf(ctx SpecContext, werfProject *werf.Project, imageName string, envs []string) string { + out := werfProject.RunCommand(ctx, []string{"stage", "image", imageName, "--log-quiet"}, werf.CommonOptions{Envs: envs}) + + lines := strings.Split(strings.TrimSpace(out), "\n") + ref := strings.TrimSpace(lines[len(lines)-1]) + Expect(ref).NotTo(BeEmpty(), "expected a stage image reference for %q, got output:\n%s", imageName, out) + + tag := ref[strings.LastIndex(ref, ":")+1:] + Expect(tag).NotTo(BeEmpty(), "expected a tag in the stage image reference %q", ref) + Expect(tag).NotTo(ContainSubstring("@"), "expected a tagged stage image reference, got %q", ref) + + return tag +} diff --git a/test/e2e/sbom/final_repo_dependent_test.go b/test/e2e/sbom/final_repo_dependent_test.go new file mode 100644 index 0000000000..b31f6e39bd --- /dev/null +++ b/test/e2e/sbom/final_repo_dependent_test.go @@ -0,0 +1,122 @@ +package e2e_build_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/werf" +) + +// The fixtures build from scratch on purpose: an image derived from a trusted +// builder base inherits the io.deckhouse.internal.builder label, and together +// with WERF_E2E_ALLOW_LOCAL_BUILDER_IMAGES a failed base SBOM lookup silently +// degrades into ErrSbomNotRequired instead of failing the build. A scratch +// base leaves no such escape, so these specs actually falsify the base/import +// SBOM lookup. +var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", "final-repo", "dependent"), func() { + It("image built from another image of the project: build with --final-repo succeeds", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_dependent" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_dependent") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + // The build itself is the primary assertion: SBOM convergence of the + // dependent image has to find the SBOM of its base image, so a lookup + // pointed at a repository that does not hold it fails the build. + By("building the two dependent images with --final-repo against a clean registry") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_dependent.json"), nil) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + appDigest := appRecord.DockerImageDigest + Expect(appDigest).NotTo(BeEmpty()) + + assertAppSbomInFinalRepo(ctx, werfProject, finalRepo, appDigest) + assertAppSbomInStagesRepo(ctx, werfProject, stagesRepo, "app") + + By("rebuilding and checking the SBOMs are served from cache, not regenerated") + rebuildOut := werfProject.Build(ctx, nil) + Expect(strings.Count(rebuildOut, "Use previously generated SBOM from registry")).To(BeNumerically(">=", 2), + "both the base and the dependent image SBOMs must be reused on rebuild") + }) + + It("image importing files from another image of the project: build with --final-repo succeeds", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_import" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_import") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + // SBOM convergence of the importing image has to find the SBOM of the + // import source, exercising the import-side lookup the same way the + // fromImage spec exercises the base-image one. + By("building the importing image with --final-repo against a clean registry") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_import.json"), nil) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + appDigest := appRecord.DockerImageDigest + Expect(appDigest).NotTo(BeEmpty()) + + assertAppSbomInFinalRepo(ctx, werfProject, finalRepo, appDigest) + assertAppSbomInStagesRepo(ctx, werfProject, stagesRepo, "app") + }) +}) + +// assertAppSbomInFinalRepo checks the final repo serves the SBOM of that very image: +// the artifact index entry has to name the image and its in-toto subject has to be +// the digest the artifact is attached to, so an SBOM of another image of the same +// project — the base image is the one at hand — does not satisfy the assertion. +func assertAppSbomInFinalRepo(ctx SpecContext, werfProject *werf.Project, finalRepo, digest string) { + By("reading the dependent image's SBOM from the final repo") + + desc, payload := fetchSingleSbomArtifact(ctx, finalRepo, digest) + Expect(desc.ArtifactType).To(Equal(attestation.DSSEMediaType)) + Expect(desc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + Expect(mustExtractInTotoSubjectDigest(payload)).To(Equal(digest)) + + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", finalRepo, "--digest", digest}, + }, + }) + sbomtest.MustParseSBOMOutput(sbomOut) +} + +// assertAppSbomInStagesRepo reads the same SBOM out of the repository the image was +// built in, addressed by its stage tag: the presence of a copy in the final repo must +// not stand in for its absence here. +func assertAppSbomInStagesRepo(ctx SpecContext, werfProject *werf.Project, stagesRepo, imageName string) { + By("reading the same image's SBOM from the stages repo") + + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTagOf(ctx, werfProject, imageName, nil)}, + }, + }) + sbomtest.MustParseSBOMOutput(sbomOut) +} diff --git a/test/e2e/sbom/final_repo_multiplatform_test.go b/test/e2e/sbom/final_repo_multiplatform_test.go new file mode 100644 index 0000000000..bb3e227ff8 --- /dev/null +++ b/test/e2e/sbom/final_repo_multiplatform_test.go @@ -0,0 +1,89 @@ +package e2e_build_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM final repo multi-platform", Label("e2e", "sbom", "final-repo", "multiplatform"), func() { + It("build with --final-repo → per-platform SBOMs on platform manifest digests in both repos, none on the index", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + SuiteData.Stubs.SetEnv("WERF_ENABLE_REPORT_BY_PLATFORM", "1") + SuiteData.Stubs.SetEnv("WERF_EXPERIMENTAL_STAPEL_ARM", "1") + + repoDirname := "repo_sbom_final_repo_multiplatform" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_multiplatform") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + By("building the multi-platform image with --final-repo") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_multiplatform.json"), nil) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo), + "expected build report to reference the final repo") + indexDigest := appRecord.DockerImageDigest + Expect(indexDigest).NotTo(BeEmpty()) + + byPlatform := buildReport.ImagesByPlatform["app"] + Expect(byPlatform).To(HaveLen(len(multiplatformSbomPlatforms)), "expected a build report record per platform") + + // The registry-level index copy into the final repo preserves the digests + // of the platform manifests it references, so the same platform digest + // addresses the manifest in both repositories. + By("verifying each platform manifest carries its own SBOM in both repositories") + for _, platform := range multiplatformSbomPlatforms { + record, hasRecord := byPlatform[platform] + Expect(hasRecord).To(BeTrue(), "no build report record for platform %s", platform) + + platformDigest := record.DockerImageDigest + Expect(platformDigest).NotTo(BeEmpty()) + Expect(platformDigest).NotTo(Equal(indexDigest)) + + for _, repo := range []string{stagesRepo, finalRepo} { + desc, payload := fetchSingleSbomArtifact(ctx, repo, platformDigest) + Expect(desc.ArtifactType).To(Equal(attestation.DSSEMediaType)) + Expect(desc.Annotations[image.WerfPlatformAnnotation]).To(Equal(platform), + "artifact attached to %s in %s must belong to platform %s", platformDigest, repo, platform) + Expect(desc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + // The artifact describes the platform manifest it is attached to: a copy + // that moved a platform SBOM onto another digest, or the wrong platform's + // SBOM onto this one, differs here and nowhere else. + Expect(mustExtractInTotoSubjectDigest(payload)).To(Equal(platformDigest), + "in-toto subject of the SBOM in %s must name the platform manifest it describes", repo) + } + } + + By("verifying no SBOM artifact is attached to the index digest in either repository") + expectNoSbomArtifact(ctx, stagesRepo, indexDigest) + expectNoSbomArtifact(ctx, finalRepo, indexDigest) + + By("reading a platform SBOM from the final repo through the digest reported to the user") + for _, platform := range multiplatformSbomPlatforms { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{ + "--repo", finalRepo, + "--digest", indexDigest, + "--platform", platform, + }, + }, + }) + sbomtest.MustParseSBOMOutput(sbomOut) + } + }) +}) diff --git a/test/e2e/sbom/final_repo_sweep_test.go b/test/e2e/sbom/final_repo_sweep_test.go new file mode 100644 index 0000000000..f42d7e97f1 --- /dev/null +++ b/test/e2e/sbom/final_repo_sweep_test.go @@ -0,0 +1,122 @@ +package e2e_build_test + +import ( + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/vex" + "github.com/werf/werf/v3/test/pkg/attestutils" + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM and VEX across every repository of a build", Label("e2e", "sbom", "final-repo", "sweep"), func() { + It("reuses stages from a secondary repo and still serves SBOM and VEX from the stages and the final repo", func(ctx SpecContext) { + setupSbomBuildEnv() + + repoDirname := "repo_sbom_final_repo_sweep" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_sweep") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + for _, envVar := range buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-sweep-builder") { + key, value, found := strings.Cut(envVar, "=") + Expect(found).To(BeTrue()) + SuiteData.Stubs.SetEnv(key, value) + } + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + + secondaryRepo := suite_init.TestRepo(SuiteData.ProjectName + "-secondary") + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName + "-stages") + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + cacheRepo := suite_init.TestRepo(SuiteData.ProjectName + "-cache") + + By("seeding a secondary repo with a full build") + SuiteData.Stubs.SetEnv("WERF_REPO", secondaryRepo) + werfProject.Build(ctx, nil) + + By("rebuilding into a clean stages repo, reusing the secondary repo and publishing to a final repo") + SuiteData.Stubs.SetEnv("WERF_REPO", stagesRepo) + SuiteData.Stubs.SetEnv("WERF_SECONDARY_REPO_1", secondaryRepo) + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + SuiteData.Stubs.SetEnv("WERF_CACHE_REPO_1", cacheRepo) + + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_sweep.json"), nil) + + for _, imageName := range []string{"base", "carrier", "app"} { + record, found := buildReport.Images[imageName] + Expect(found).To(BeTrue(), "expected image %q in build report", imageName) + Expect(record.StagesSkipped).To(BeTrue(), + "image %q must be reused from the secondary repo by content-based tag, not rebuilt", imageName) + } + + appRecord := buildReport.Images["app"] + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + appDigest := appRecord.DockerImageDigest + Expect(appDigest).NotTo(BeEmpty()) + + By("the final repo serves this image's own SBOM, and it carries what the base and the import source contributed") + desc, payload := fetchSingleSbomArtifact(ctx, finalRepo, appDigest) + Expect(desc.ArtifactType).To(Equal(attestation.DSSEMediaType)) + Expect(desc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + Expect(mustExtractInTotoSubjectDigest(payload)).To(Equal(appDigest)) + + finalSbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"--repo", finalRepo, "--digest", appDigest}}, + }) + finalBom := sbomtest.MustParseSBOMOutput(finalSbomOut) + assertCarriesImportedComponents(ctx, werfProject, stagesRepo, finalBom) + + By("the stages repo serves the same SBOM on its own, not by proxy of the final repo") + stagesSbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTagOf(ctx, werfProject, "app", nil)}}, + }) + assertCarriesImportedComponents(ctx, werfProject, stagesRepo, sbomtest.MustParseSBOMOutput(stagesSbomOut)) + + By("VEX is available next to the image in both the stages and the final repo") + Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, finalRepo, appDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), + "the final repo must serve the VEX document of the image it publishes") + + stagesDigest := stagesImageDigestOf(ctx, werfProject, "app") + Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, stagesRepo, stagesDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), + "the stages repo must serve the VEX document on its own") + + By("the cache repo holds the stage under the stages repo digest and serves its SBOM and VEX") + cacheDesc, cachePayload := fetchSingleSbomArtifact(ctx, cacheRepo, stagesDigest) + Expect(cacheDesc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + Expect(mustExtractInTotoSubjectDigest(cachePayload)).To(Equal(stagesDigest)) + Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, cacheRepo, stagesDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), + "the cache repo must serve the VEX document of the stage it mirrors") + }) +}) + +// assertCarriesImportedComponents checks the SBOM of app contains everything the +// SBOMs of its base image and its import source list. app is a Stapel image with +// no packages directive, so it is never scanned on its own: a component of either +// source can only have reached it through the merge of that image's SBOM. base +// (Go module) and carrier (Cargo) catalog different ecosystems, so each side of +// the merge is discriminated on its own. +func assertCarriesImportedComponents(ctx SpecContext, werfProject *werf.Project, stagesRepo string, appBom *cdx.BOM) { + for _, source := range []string{"base", "carrier"} { + sourceSbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTagOf(ctx, werfProject, source, nil)}}, + }) + sourceBom := sbomtest.MustParseSBOMOutput(sourceSbomOut) + + Expect(sourceBom.Components).NotTo(BeNil()) + Expect(*sourceBom.Components).NotTo(BeEmpty(), "image %q must contribute components for this assertion to discriminate", source) + + for _, component := range *sourceBom.Components { + Expect(sbomtest.FindComponentByPURL(appBom, component.PackageURL)).NotTo(BeNil(), + "component %s of image %q is missing from the merged SBOM", component.PackageURL, source) + } + } +} diff --git a/test/e2e/sbom/helpers_test.go b/test/e2e/sbom/helpers_test.go index 21968bd186..33a9155370 100644 --- a/test/e2e/sbom/helpers_test.go +++ b/test/e2e/sbom/helpers_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "io" + "os" "slices" "github.com/google/go-containerregistry/pkg/authn" @@ -17,6 +18,7 @@ import ( "github.com/werf/werf/v3/pkg/attestation" "github.com/werf/werf/v3/pkg/oci/artifact" sbomImage "github.com/werf/werf/v3/pkg/sbom/image" + "github.com/werf/werf/v3/test/pkg/werf" ) var multiplatformSbomPlatforms = []string{"linux/amd64", "linux/arm64"} @@ -95,6 +97,21 @@ func expectNoSbomArtifact(ctx SpecContext, repo, parentDigest string) { Expect(dsseDescs).To(BeEmpty(), "no SBOM artifact must be attached to the index digest %s", parentDigest) } +// stagesImageDigestOf resolves the digest of the image's last stage in the stages +// repo, which is what artifacts in the stages and cache repos are attached to. +func stagesImageDigestOf(ctx SpecContext, werfProject *werf.Project, imageName string) string { + stagesRepo := os.Getenv("WERF_REPO") + Expect(stagesRepo).NotTo(BeEmpty()) + + tagRef, err := name.NewTag(stagesRepo+":"+stageTagOf(ctx, werfProject, imageName, nil), name.Insecure) + Expect(err).NotTo(HaveOccurred()) + + desc, err := remote.Get(tagRef, insecureRemoteOptions(ctx)...) + Expect(err).NotTo(HaveOccurred()) + + return desc.Digest.String() +} + func mustExtractInTotoSubjectDigest(dsseEnvelope []byte) string { var envelope struct { Payload []byte `json:"payload"`