From f517dfb21261493eb39e0b2cbf2fca46e965c104 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Tue, 1 Sep 2026 10:49:12 +0300 Subject: [PATCH 01/11] test(sbom, cleanup): cover SBOM retention across cleanup werf cleanup deletes stages from the final repo and then collects orphaned sha256-* artifact indexes there, but no e2e run exercised that path against a live SBOM. The existing final-repo test stops right after the build, and the cleanup case in the VEX suite asserts deletion in the stages repo, so a regression that dropped an in-use SBOM from the final repo would go unnoticed. Build a project with --final-repo, read the SBOM back from the final repo and from the stages repo, run cleanup twice with the built commit reachable from origin so retention policies hold the stage, and read the SBOM again after each run. Address the stages repo by its stage tag rather than by the digest from the build report, because copying a stage into the final repo may change the manifest digest. Keep the bare remote outside the work tree: the fixture adds the whole project directory to the image, so an in-tree remote breaks giterminism. Signed-off-by: Radmir Khurum --- test/e2e/sbom/final_repo_cleanup_test.go | 116 +++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 test/e2e/sbom/final_repo_cleanup_test.go diff --git a/test/e2e/sbom/final_repo_cleanup_test.go b/test/e2e/sbom/final_repo_cleanup_test.go new file mode 100644 index 0000000000..2636de0bd5 --- /dev/null +++ b/test/e2e/sbom/final_repo_cleanup_test.go @@ -0,0 +1,116 @@ +package e2e_build_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/utils" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM retention across cleanup", Label("e2e", "sbom", "final-repo", "cleanup"), func() { + DescribeTable("build with --final-repo → cleanup → SBOM still readable from both repos", + func(ctx SpecContext, testOpts sbomTestOptions) { + setupSbomBuildEnv(testOpts.setupEnvOptions) + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_cleanup" + SuiteData.InitTestRepo(ctx, repoDirname, "inject/ospm_basic") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + // werf cleanup keeps an image only while the commit it was built from stays + // reachable from a remote branch. Without origin every image looks unreachable + // and gets deleted, which would hide the retention this test asserts. + // The bare remote lives outside the work tree: the fixture adds the whole + // project directory to the image, so an in-tree remote would break giterminism. + remotePath := SuiteData.GetTestRepoPath(repoDirname + "_remote.git") + utils.RunSucceedCommand(ctx, testRepoPath, "git", "init", "--bare", remotePath) + utils.RunSucceedCommand(ctx, testRepoPath, "git", "remote", "add", "origin", remotePath) + utils.RunSucceedCommand(ctx, testRepoPath, "git", "push", "--set-upstream", "origin", "HEAD:refs/heads/main") + + builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-cleanup-builder") + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_cleanup.json"), + &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, + ) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo), + "expected build report to reference the final repo") + finalDigest := appRecord.DockerImageDigest + Expect(finalDigest).NotTo(BeEmpty()) + + // The stage copy into the final repo may change the manifest digest, so the + // stages repo has to be addressed by its own stage tag rather than by finalDigest. + stageTag := stageTagOf(ctx, werfProject, "app", builderEnv) + + assertSbomInFinalRepo := func() { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", finalRepo, "--digest", finalDigest}, + Envs: builderEnv, + }, + }) + sbomtest.AssertHasComponent(sbomtest.MustParseSBOMOutput(sbomOut), "curl", "8.12.1") + } + + assertSbomInStagesRepo := func() { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTag}, + Envs: builderEnv, + }, + }) + sbomtest.AssertHasComponent(sbomtest.MustParseSBOMOutput(sbomOut), "curl", "8.12.1") + } + + By("reading the SBOM from both repos before cleanup") + assertSbomInFinalRepo() + assertSbomInStagesRepo() + + By("running cleanup while the image is still in use") + werfProject.RunCommand(ctx, []string{"cleanup", "--without-kube"}, werf.CommonOptions{Envs: builderEnv}) + + By("reading the SBOM from both repos after cleanup") + assertSbomInFinalRepo() + assertSbomInStagesRepo() + + By("running cleanup a second time") + werfProject.RunCommand(ctx, []string{"cleanup", "--without-kube"}, werf.CommonOptions{Envs: builderEnv}) + + By("reading the SBOM from both repos after the repeated cleanup") + assertSbomInFinalRepo() + assertSbomInStagesRepo() + }, + Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), + Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), + ) +}) + +// stageTagOf returns the content-based tag of the image's last stage in the stages repo. +// werf stage image prints the reference as its last line, so the preceding log lines are dropped. +func stageTagOf(ctx SpecContext, werfProject *werf.Project, imageName string, envs []string) string { + out := werfProject.RunCommand(ctx, []string{"stage", "image", imageName, "--log-quiet"}, werf.CommonOptions{Envs: envs}) + + lines := strings.Split(strings.TrimSpace(out), "\n") + ref := strings.TrimSpace(lines[len(lines)-1]) + Expect(ref).NotTo(BeEmpty(), "expected a stage image reference for %q, got output:\n%s", imageName, out) + + parts := strings.Split(ref, ":") + tag := parts[len(parts)-1] + Expect(tag).NotTo(BeEmpty(), "expected a tag in the stage image reference %q", ref) + + return tag +} From 46e8cc5bcc5c49864eac9563e7c558da029b1a57 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Wed, 2 Sep 2026 09:33:55 +0300 Subject: [PATCH 02/11] test(sbom): cover dependent images and multi-platform placement with final repo Two gaps around --final-repo had no coverage. A project where one image is the base of another (fromImage) or imports files from another exercises the base/import SBOM lookup during convergence of the dependent image. If that lookup resolves against a repository that does not hold the base SBOM, the build fails with advice to rebuild with SBOM generation enabled, which cannot help. Cover both dependency kinds: build with --final-repo against a clean registry, read the merged SBOM from the final repo, and check a rebuild serves both SBOMs from cache instead of regenerating them. For a multi-platform image the SBOMs belong on the platform manifest digests, never on the index digest, and the registry-level index copy into the final repo preserves the platform manifest digests. Assert each platform manifest carries its SBOM in both the stages repo and the final repo, the index digest carries none in either, and sbom get resolves a platform SBOM from the final repo through the index digest reported to the user. The stapel fixture builds for both platforms on the Docker backend with WERF_EXPERIMENTAL_STAPEL_ARM, following the multi-platform signing suite. Signed-off-by: Radmir Khurum --- .../final_repo_import/Dockerfile.builder-base | 3 + .../final_repo_import/werf-giterminism.yaml | 5 + .../_fixtures/final_repo_import/werf.yaml | 27 +++++ test/e2e/sbom/final_repo_dependent_test.go | 105 ++++++++++++++++++ .../e2e/sbom/final_repo_multiplatform_test.go | 85 ++++++++++++++ 5 files changed, 225 insertions(+) create mode 100644 test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base create mode 100644 test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml create mode 100644 test/e2e/sbom/_fixtures/final_repo_import/werf.yaml create mode 100644 test/e2e/sbom/final_repo_dependent_test.go create mode 100644 test/e2e/sbom/final_repo_multiplatform_test.go diff --git a/test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base b/test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base new file mode 100644 index 0000000000..770edcb4a5 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base @@ -0,0 +1,3 @@ +FROM registry.deckhouse.io/container-factory@sha256:b9ebf99d849cc88889ee2281f8a9100fdeee4b95b11c4a0350f38135a835f5d1 +LABEL io.deckhouse.internal.builder=true +ENV PACKAGES_VERSION=v1.3.6 diff --git a/test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml b/test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml new file mode 100644 index 0000000000..9483c3670c --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml @@ -0,0 +1,5 @@ +giterminismConfigVersion: 1 +config: + goTemplateRendering: + allowEnvVariables: + - BUILDER_BASE_IMAGE diff --git a/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml new file mode 100644 index 0000000000..e8980a5635 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml @@ -0,0 +1,27 @@ +project: werf-test-e2e-sbom-final-repo-import +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: carrier +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: / + to: / +packages: + - type: os-pm + spec: + - jq==1.8.1 +--- +image: app +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: / + to: /app +import: + - image: carrier + add: /werf.yaml + to: /imported/werf.yaml + before: install diff --git a/test/e2e/sbom/final_repo_dependent_test.go b/test/e2e/sbom/final_repo_dependent_test.go new file mode 100644 index 0000000000..e91ff422a8 --- /dev/null +++ b/test/e2e/sbom/final_repo_dependent_test.go @@ -0,0 +1,105 @@ +package e2e_build_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", "final-repo", "dependent"), func() { + DescribeTable("image built from another image of the project: build with --final-repo succeeds and merges the base SBOM", + func(ctx SpecContext, testOpts sbomTestOptions) { + setupSbomBuildEnv(testOpts.setupEnvOptions) + + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_dependent" + SuiteData.InitTestRepo(ctx, repoDirname, "packages_merge/base_with_child") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-dependent-builder") + + // The build itself is the primary assertion: SBOM convergence of the + // dependent image has to find the SBOM of its base image, so a lookup + // pointed at a repository that does not hold it fails the build. + By("building the two dependent images with --final-repo against a clean registry") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_dependent.json"), + &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, + ) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + Expect(appRecord.DockerImageDigest).NotTo(BeEmpty()) + + By("reading the dependent image's SBOM and checking the base image contribution") + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", finalRepo, "--digest", appRecord.DockerImageDigest}, + Envs: builderEnv, + }, + }) + bom := sbomtest.MustParseSBOMOutput(sbomOut) + sbomtest.AssertHasComponent(bom, "jq", "1.8.1") + sbomtest.AssertHasComponent(bom, "curl", "8.12.1") + + By("rebuilding and checking the SBOMs are served from cache, not regenerated") + rebuildOut := werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}) + Expect(strings.Count(rebuildOut, "Use previously generated SBOM from registry")).To(BeNumerically(">=", 2), + "both the base and the dependent image SBOMs must be reused on rebuild") + }, + Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), + Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), + ) + + DescribeTable("image importing files from another image of the project: build with --final-repo succeeds", + func(ctx SpecContext, testOpts sbomTestOptions) { + setupSbomBuildEnv(testOpts.setupEnvOptions) + + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_import" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_import") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-import-builder") + + // SBOM convergence of the importing image has to find the SBOM of the + // import source, exercising the import-side lookup the same way the + // fromImage table exercises the base-image one. + By("building the importing image with --final-repo against a clean registry") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_import.json"), + &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, + ) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerImageDigest).NotTo(BeEmpty()) + + By("reading the importing image's SBOM") + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", finalRepo, "--digest", appRecord.DockerImageDigest}, + Envs: builderEnv, + }, + }) + sbomtest.MustParseSBOMOutput(sbomOut) + }, + Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), + Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), + ) +}) diff --git a/test/e2e/sbom/final_repo_multiplatform_test.go b/test/e2e/sbom/final_repo_multiplatform_test.go new file mode 100644 index 0000000000..84eec68be1 --- /dev/null +++ b/test/e2e/sbom/final_repo_multiplatform_test.go @@ -0,0 +1,85 @@ +package e2e_build_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM final repo multi-platform", Label("e2e", "sbom", "final-repo", "multiplatform"), func() { + DescribeTable("build with --final-repo → per-platform SBOMs on platform manifest digests in both repos, none on the index", + func(ctx SpecContext, testOpts sbomTestOptions) { + setupSbomBuildEnv(testOpts.setupEnvOptions) + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + SuiteData.Stubs.SetEnv("WERF_ENABLE_REPORT_BY_PLATFORM", "1") + SuiteData.Stubs.SetEnv("WERF_EXPERIMENTAL_STAPEL_ARM", "1") + + repoDirname := "repo_sbom_final_repo_multiplatform" + SuiteData.InitTestRepo(ctx, repoDirname, "multiplatform") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + By("building the multi-platform image with --final-repo") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_multiplatform.json"), nil) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo), + "expected build report to reference the final repo") + indexDigest := appRecord.DockerImageDigest + Expect(indexDigest).NotTo(BeEmpty()) + + byPlatform := buildReport.ImagesByPlatform["app"] + Expect(byPlatform).To(HaveLen(len(multiplatformSbomPlatforms)), "expected a build report record per platform") + + // The registry-level index copy into the final repo preserves the digests + // of the platform manifests it references, so the same platform digest + // addresses the manifest in both repositories. + By("verifying each platform manifest carries its SBOM in both repositories") + for _, platform := range multiplatformSbomPlatforms { + record, hasRecord := byPlatform[platform] + Expect(hasRecord).To(BeTrue(), "no build report record for platform %s", platform) + + platformDigest := record.DockerImageDigest + Expect(platformDigest).NotTo(BeEmpty()) + Expect(platformDigest).NotTo(Equal(indexDigest)) + + stagesDesc, _ := fetchSingleSbomArtifact(ctx, stagesRepo, platformDigest) + Expect(stagesDesc.Annotations[image.WerfPlatformAnnotation]).To(Equal(platform)) + + finalDesc, _ := fetchSingleSbomArtifact(ctx, finalRepo, platformDigest) + Expect(finalDesc.Annotations[image.WerfPlatformAnnotation]).To(Equal(platform)) + } + + By("verifying no SBOM artifact is attached to the index digest in either repository") + expectNoSbomArtifact(ctx, stagesRepo, indexDigest) + expectNoSbomArtifact(ctx, finalRepo, indexDigest) + + By("reading a platform SBOM from the final repo through the digest reported to the user") + for _, platform := range multiplatformSbomPlatforms { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{ + "--repo", finalRepo, + "--digest", indexDigest, + "--platform", platform, + }, + }, + }) + sbomtest.MustParseSBOMOutput(sbomOut) + } + }, + Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), + Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), + ) +}) From 14c5f6c15eec5429ac09e733d23ef231b8b67671 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Wed, 2 Sep 2026 17:30:58 +0300 Subject: [PATCH 03/11] test(sbom): make dependent final-repo entries falsify the base SBOM lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuild the dependent-images fixtures from scratch instead of the trusted builder base. An image derived from the builder base inherits the io.deckhouse.internal.builder label, and with WERF_E2E_ALLOW_LOCAL_BUILDER_IMAGES a failed base SBOM lookup silently degrades into ErrSbomNotRequired while the asserted base packages surface through the filesystem scan — the previous entries passed without exercising the lookup at all. The scratch fixtures leave no escape: the fromImage entries now fail on current main, reproducing the broken base SBOM lookup under --final-repo, and need no builder base or external environment beyond the registry. Signed-off-by: Radmir Khurum --- .../_fixtures/final_repo_dependent/werf.yaml | 18 ++++++++++ .../final_repo_import/Dockerfile.builder-base | 3 -- .../final_repo_import/werf-giterminism.yaml | 5 --- .../_fixtures/final_repo_import/werf.yaml | 16 ++++----- test/e2e/sbom/final_repo_dependent_test.go | 34 ++++++++----------- 5 files changed, 38 insertions(+), 38 deletions(-) create mode 100644 test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml delete mode 100644 test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base delete mode 100644 test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml diff --git a/test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml new file mode 100644 index 0000000000..f5189f97b9 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_dependent/werf.yaml @@ -0,0 +1,18 @@ +project: werf-test-e2e-sbom-final-repo-dependent +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: base +from: registry.werf.io/werf/scratch:latest +git: + - add: / + to: /b +--- +image: app +fromImage: base +git: + - add: / + to: /a diff --git a/test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base b/test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base deleted file mode 100644 index 770edcb4a5..0000000000 --- a/test/e2e/sbom/_fixtures/final_repo_import/Dockerfile.builder-base +++ /dev/null @@ -1,3 +0,0 @@ -FROM registry.deckhouse.io/container-factory@sha256:b9ebf99d849cc88889ee2281f8a9100fdeee4b95b11c4a0350f38135a835f5d1 -LABEL io.deckhouse.internal.builder=true -ENV PACKAGES_VERSION=v1.3.6 diff --git a/test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml b/test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml deleted file mode 100644 index 9483c3670c..0000000000 --- a/test/e2e/sbom/_fixtures/final_repo_import/werf-giterminism.yaml +++ /dev/null @@ -1,5 +0,0 @@ -giterminismConfigVersion: 1 -config: - goTemplateRendering: - allowEnvVariables: - - BUILDER_BASE_IMAGE diff --git a/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml index e8980a5635..4b2d999412 100644 --- a/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml +++ b/test/e2e/sbom/_fixtures/final_repo_import/werf.yaml @@ -6,22 +6,18 @@ build: standard: cyclonedx@1.6 --- image: carrier -from: {{ env "BUILDER_BASE_IMAGE" }} +from: registry.werf.io/werf/scratch:latest git: - add: / - to: / -packages: - - type: os-pm - spec: - - jq==1.8.1 + to: /c --- image: app -from: {{ env "BUILDER_BASE_IMAGE" }} +from: registry.werf.io/werf/scratch:latest git: - add: / - to: /app + to: /a import: - image: carrier - add: /werf.yaml + add: /c/werf.yaml to: /imported/werf.yaml - before: install + before: setup diff --git a/test/e2e/sbom/final_repo_dependent_test.go b/test/e2e/sbom/final_repo_dependent_test.go index e91ff422a8..96cca8f3ab 100644 --- a/test/e2e/sbom/final_repo_dependent_test.go +++ b/test/e2e/sbom/final_repo_dependent_test.go @@ -12,8 +12,14 @@ import ( "github.com/werf/werf/v3/test/pkg/werf" ) +// The fixtures build from scratch on purpose: an image derived from a trusted +// builder base inherits the io.deckhouse.internal.builder label, and together +// with WERF_E2E_ALLOW_LOCAL_BUILDER_IMAGES a failed base SBOM lookup silently +// degrades into ErrSbomNotRequired instead of failing the build. A scratch +// base leaves no such escape, so these tables actually falsify the base/import +// SBOM lookup. var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", "final-repo", "dependent"), func() { - DescribeTable("image built from another image of the project: build with --final-repo succeeds and merges the base SBOM", + DescribeTable("image built from another image of the project: build with --final-repo succeeds", func(ctx SpecContext, testOpts sbomTestOptions) { setupSbomBuildEnv(testOpts.setupEnvOptions) @@ -21,11 +27,9 @@ var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", " SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) repoDirname := "repo_sbom_final_repo_dependent" - SuiteData.InitTestRepo(ctx, repoDirname, "packages_merge/base_with_child") + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_dependent") testRepoPath := SuiteData.GetTestRepoPath(repoDirname) - builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-dependent-builder") - // The build itself is the primary assertion: SBOM convergence of the // dependent image has to find the SBOM of its base image, so a lookup // pointed at a repository that does not hold it fails the build. @@ -33,28 +37,23 @@ var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", " werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) reportProject := report.NewProjectWithReport(werfProject) _, buildReport := reportProject.BuildWithReport(ctx, - SuiteData.GetBuildReportPath("sbom_final_repo_dependent.json"), - &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, - ) + SuiteData.GetBuildReportPath("sbom_final_repo_dependent.json"), nil) appRecord, found := buildReport.Images["app"] Expect(found).To(BeTrue(), "expected image %q in build report", "app") Expect(appRecord.DockerRepo).To(Equal(finalRepo)) Expect(appRecord.DockerImageDigest).NotTo(BeEmpty()) - By("reading the dependent image's SBOM and checking the base image contribution") + By("reading the dependent image's SBOM from the final repo") sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ CommonOptions: werf.CommonOptions{ ExtraArgs: []string{"--repo", finalRepo, "--digest", appRecord.DockerImageDigest}, - Envs: builderEnv, }, }) - bom := sbomtest.MustParseSBOMOutput(sbomOut) - sbomtest.AssertHasComponent(bom, "jq", "1.8.1") - sbomtest.AssertHasComponent(bom, "curl", "8.12.1") + sbomtest.MustParseSBOMOutput(sbomOut) By("rebuilding and checking the SBOMs are served from cache, not regenerated") - rebuildOut := werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}) + rebuildOut := werfProject.Build(ctx, nil) Expect(strings.Count(rebuildOut, "Use previously generated SBOM from registry")).To(BeNumerically(">=", 2), "both the base and the dependent image SBOMs must be reused on rebuild") }, @@ -73,8 +72,6 @@ var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", " SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_import") testRepoPath := SuiteData.GetTestRepoPath(repoDirname) - builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-import-builder") - // SBOM convergence of the importing image has to find the SBOM of the // import source, exercising the import-side lookup the same way the // fromImage table exercises the base-image one. @@ -82,19 +79,16 @@ var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", " werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) reportProject := report.NewProjectWithReport(werfProject) _, buildReport := reportProject.BuildWithReport(ctx, - SuiteData.GetBuildReportPath("sbom_final_repo_import.json"), - &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, - ) + SuiteData.GetBuildReportPath("sbom_final_repo_import.json"), nil) appRecord, found := buildReport.Images["app"] Expect(found).To(BeTrue(), "expected image %q in build report", "app") Expect(appRecord.DockerImageDigest).NotTo(BeEmpty()) - By("reading the importing image's SBOM") + By("reading the importing image's SBOM from the final repo") sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ CommonOptions: werf.CommonOptions{ ExtraArgs: []string{"--repo", finalRepo, "--digest", appRecord.DockerImageDigest}, - Envs: builderEnv, }, }) sbomtest.MustParseSBOMOutput(sbomOut) From 420b5c8a724934e3a2002adf27b9ae216f0b1487 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Thu, 3 Sep 2026 09:26:11 +0300 Subject: [PATCH 04/11] fix(sbom, vex, build): keep artifacts with the image in every repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With --final-repo the SBOM and VEX of an image existed in exactly one copy, in the final repo: publishFinalImage overwrote the image's content tag descriptor with the final repo one, artifact convergence resolved its target through that descriptor, and the propagation step into the final repo had been dead since the descriptor stopped being stored where the step read it. Reading the SBOM from the stages repo returned nothing, a project where one final image is built from another failed with advice to enable SBOM generation that was already enabled, and for a multi-platform image the artifacts were copied onto the index digest — where no retrieval path looks and where the second platform's copy displaced the first. Split the two roles of the content tag descriptor: contentTagDesc keeps describing the image in the repository it was built in and is the single target for artifact convergence and for the base and import SBOM lookups, while the new finalContentTagDesc carries the published final repo descriptor for the build report. Replace the per-platform SBOM propagation with one step that runs after both SBOM and VEX convergence and carries every attached artifact kind: for a single-platform image onto the published final descriptor, for a multi-platform image onto the platform manifest digests preserved by the registry-level index copy, with image-level artifacts following the index digest. The step runs on every build and the copies are idempotent, so a repository holding the image without its artifacts is repaired by the next run. Extend the same contract to the remaining copy paths: stage copies between repositories carry the artifacts of every manifest an index references and repair a destination that already holds the manifest, werf stages copy and bundle copy carry the artifacts of the images they transfer, and the backend-mediated secondary-to-primary copy carries artifacts only when the digest survived the copy — a statement about the source digest is not a statement about the destination digest — leaving a warning about what was left behind otherwise. Signed-off-by: Radmir Khurum --- pkg/build/build_phase.go | 97 +++++++++++++++++++++----- pkg/build/build_phase_test.go | 8 --- pkg/build/build_report.go | 2 +- pkg/build/image/image.go | 27 ++++++- pkg/build/sbom_step.go | 24 +++---- pkg/build/sbom_step_propagate_test.go | 23 ++---- pkg/build/stages/remote_storage.go | 11 +++ pkg/deploy/bundles/copy_test.go | 8 +++ pkg/deploy/bundles/remote_bundle.go | 10 +++ pkg/oci/artifact/copy.go | 28 ++++++++ pkg/storage/manager/storage_manager.go | 15 +++- pkg/storage/repo_stages_storage.go | 9 ++- 12 files changed, 200 insertions(+), 62 deletions(-) diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index bafa3391f2..2cde9efa23 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -510,6 +510,10 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error { return err } + if err := phase.propagateArtifacts(ctx); err != nil { + return err + } + telemetry.GetTelemetryWerfIO().BuildFinished(ctx, true) return phase.createReport(ctx, imagesPairs) @@ -603,9 +607,8 @@ func (phase *BuildPhase) convergeImageSbom(ctx context.Context, name string, ima signerIdentity = phase.SbomSigningOptions.Signer().Fingerprint() } - finalStageDesc := phase.finalStageDescForImage(name, images) for _, img := range images { - if err := phase.convergePlatformImageSbom(ctx, name, img, finalStageDesc, signer, signerIdentity, breaker); err != nil { + if err := phase.convergePlatformImageSbom(ctx, name, img, signer, signerIdentity, breaker); err != nil { return err } } @@ -613,7 +616,7 @@ func (phase *BuildPhase) convergeImageSbom(ctx context.Context, name string, ima return nil } -func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name string, img *image.Image, finalStageDesc *imagePkg.StageDesc, signer signature.Signer, signerIdentity string, breaker *externalref.ResolverBreaker) error { +func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name string, img *image.Image, signer signature.Signer, signerIdentity string, breaker *externalref.ResolverBreaker) error { stageDesc := img.GetLastNonEmptyStageDesc() if stageDesc == nil { return fmt.Errorf("unable to converge sbom for image %q: stage descriptor is unavailable", name) @@ -677,25 +680,81 @@ func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name str return fmt.Errorf("unable to converge sbom for image %q: %w", name, err) } - if err := phase.sbomStep.PropagateArtifacts(ctx, name, stageDesc, finalStageDesc, phase.Conveyor.StorageManager.GetCacheStagesStorageList()); err != nil { - return fmt.Errorf("unable to propagate sbom for image %q: %w", name, err) - } - return nil } -// finalStageDescForImage returns the final repo descriptor to copy the SBOM artifacts into, or nil -// when there is nothing to copy. A single-platform image never has one: publishFinalImage stores the -// final repo descriptor in the content tag desc, which convergeImageSbom already uses as the SBOM -// target. Reaching for the last non-empty stage here instead panics, because an image resolved from -// the cache short-circuits in BeforeImageStages and never gets one. -func (phase *BuildPhase) finalStageDescForImage(name string, images []*image.Image) *imagePkg.StageDesc { - if len(images) == 1 { - return nil - } - if multiImg := phase.Conveyor.imagesTree.GetMultiplatformImage(name); multiImg != nil { - return multiImg.GetFinalStageDesc() +// propagateArtifacts copies the artifacts attached to every image in the stages repo +// into the final repo and the cache repos. It runs after both SBOM and VEX +// convergence so it carries every attached artifact kind, and it runs on every build, +// so a destination holding the image without its artifacts is repaired by the next +// run. For a multi-platform image the per-platform artifacts are copied onto the +// platform manifest digests — preserved by the registry-level index copy — and the +// image-level artifacts (e.g. VEX) onto the index digest. +func (phase *BuildPhase) propagateArtifacts(ctx context.Context) error { + cacheStagesStorageList := phase.Conveyor.StorageManager.GetCacheStagesStorageList() + + for _, pair := range phase.Conveyor.imagesTree.GetImagesByName(false) { + name, images := pair.Unpair() + + if multiImg := phase.Conveyor.imagesTree.GetMultiplatformImage(name); multiImg != nil { + var finalRepo string + finalDesc := multiImg.GetFinalStageDesc() + if finalDesc != nil { + finalRepo = finalDesc.Info.Repository + } + + for _, img := range images { + stageDesc := img.GetContentTagDesc() + if stageDesc == nil { + continue + } + if err := phase.sbomStep.PropagateArtifacts(ctx, name, + stageDesc.Info.Repository, stageDesc.Info.GetDigest(), + finalRepo, stageDesc.Info.GetDigest(), + cacheStagesStorageList, + ); err != nil { + return fmt.Errorf("unable to propagate artifacts for image %q (platform %s): %w", name, img.TargetPlatform, err) + } + } + + if idxDesc := multiImg.GetStageDesc(); idxDesc != nil { + var finalDigest string + if finalDesc != nil { + finalDigest = finalDesc.Info.GetDigest() + } + if err := phase.sbomStep.PropagateArtifacts(ctx, name, + idxDesc.Info.Repository, idxDesc.Info.GetDigest(), + finalRepo, finalDigest, + cacheStagesStorageList, + ); err != nil { + return fmt.Errorf("unable to propagate artifacts for image %q: %w", name, err) + } + } + + continue + } + + img := images[0] + stageDesc := img.GetContentTagDesc() + if stageDesc == nil { + continue + } + + var finalRepo, finalDigest string + if finalDesc := img.GetFinalContentTagDesc(); finalDesc != nil { + finalRepo = finalDesc.Info.Repository + finalDigest = finalDesc.Info.GetDigest() + } + + if err := phase.sbomStep.PropagateArtifacts(ctx, name, + stageDesc.Info.Repository, stageDesc.Info.GetDigest(), + finalRepo, finalDigest, + cacheStagesStorageList, + ); err != nil { + return fmt.Errorf("unable to propagate artifacts for image %q: %w", name, err) + } } + return nil } @@ -780,7 +839,7 @@ func (phase *BuildPhase) publishFinalImage(ctx context.Context, name string, img if err != nil { return fmt.Errorf("unable to copy image into final repo: %w", err) } - img.SetContentTagDesc(desc) + img.SetFinalContentTagDesc(desc) return nil } diff --git a/pkg/build/build_phase_test.go b/pkg/build/build_phase_test.go index b7e89929ea..00a46fb9b9 100644 --- a/pkg/build/build_phase_test.go +++ b/pkg/build/build_phase_test.go @@ -388,12 +388,4 @@ var _ = Describe("BuildPhase", func() { Expect(digestDisabledAgain).To(Equal(digestBaseline)) }) }) - - Describe("finalStageDescForImage", func() { - It("returns nil for a single-platform image resolved from the cache, without a built stage image", func() { - phase := &BuildPhase{} - - Expect(phase.finalStageDescForImage("app", []*image.Image{{}})).To(BeNil()) - }) - }) }) diff --git a/pkg/build/build_report.go b/pkg/build/build_report.go index eb1b3a4717..791791338e 100644 --- a/pkg/build/build_report.go +++ b/pkg/build/build_report.go @@ -242,7 +242,7 @@ func createBuildReport(ctx context.Context, phase *BuildPhase, imagePairs []util targetPlatforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform }) for _, img := range images { - imageDesc := img.GetContentTagDesc() + imageDesc := img.GetPublishedContentTagDesc() var stages []ReportStageRecord if !img.AnchorReused { stages = getStagesReport(img, false) diff --git a/pkg/build/image/image.go b/pkg/build/image/image.go index c8402c06bc..fddcb8c4c6 100644 --- a/pkg/build/image/image.go +++ b/pkg/build/image/image.go @@ -138,8 +138,11 @@ type Image struct { stageDurations map[stage.StageName]time.Duration lastNonEmptyStage stage.Interface contentTagDesc *image.StageDesc - rebuilt bool - useCustomTag bool + // finalContentTagDesc is the content tag descriptor of the image as copied into + // the final repo; contentTagDesc keeps describing the image in the stages repo. + finalContentTagDesc *image.StageDesc + rebuilt bool + useCustomTag bool baseImageType BaseImageType baseImageReference string @@ -344,6 +347,26 @@ func (i *Image) GetContentTagDesc() *image.StageDesc { return i.contentTagDesc } +func (i *Image) SetFinalContentTagDesc(desc *image.StageDesc) { + i.finalContentTagDesc = desc +} + +func (i *Image) GetFinalContentTagDesc() *image.StageDesc { + return i.finalContentTagDesc +} + +// GetPublishedContentTagDesc returns the descriptor of the image as published for +// consumers: the final repo one when the image was copied there, the stages repo +// one otherwise. Build-internal consumers — artifact convergence, base and import +// SBOM lookups — must keep using GetContentTagDesc, which always describes the +// image in the repository it was built in. +func (i *Image) GetPublishedContentTagDesc() *image.StageDesc { + if i.finalContentTagDesc != nil { + return i.finalContentTagDesc + } + return i.contentTagDesc +} + func (i *Image) GetStage(name stage.StageName) stage.Interface { for _, s := range i.stages { if s.Name() == name { diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index 1946a0cd92..574a91dc41 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -212,18 +212,18 @@ func (step *sbomStep) calculateStableChecksum(scanOpts scanner.ScanOptions, merg ) } -// PropagateArtifacts copies the artifacts attached to the image stage (e.g. its SBOM) -// into the final repo and the cache repos. Stages themselves are copied there before -// SBOM generation runs, so the artifacts have to catch up separately. -func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName string, stageDesc, finalStageDesc *image.StageDesc, cacheStagesStorageList []storage.StagesStorage) error { - srcRepo := stageDesc.Info.Repository - srcDigest := stageDesc.Info.GetDigest() - - if finalStageDesc != nil && finalStageDesc.Info.Repository != srcRepo { - if err := logboek.Context(ctx).Default().LogProcess("image %s: Copy SBOM artifacts into the final repo %s", werfImgName, finalStageDesc.Info.Repository).DoError(func() error { - return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, finalStageDesc.Info.Repository, finalStageDesc.Info.GetDigest()) +// PropagateArtifacts copies the artifacts attached to the image in the repository it +// was built in — its SBOM, VEX and any other attached kind — into the final repo and +// the cache repos. Stages are copied there before the artifacts exist, so the +// artifacts have to catch up separately. The copy runs on every build and is +// idempotent, so a destination holding the image without its artifacts is repaired +// by the next run. +func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName, srcRepo, srcDigest, finalRepo, finalDigest string, cacheStagesStorageList []storage.StagesStorage) error { + if finalRepo != "" && finalRepo != srcRepo { + if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy attached artifacts into the final repo %s", werfImgName, finalRepo).DoError(func() error { + return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, finalRepo, finalDigest) }); err != nil { - return fmt.Errorf("copy attached artifacts into final repo %s: %w", finalStageDesc.Info.Repository, err) + return fmt.Errorf("copy attached artifacts into final repo %s: %w", finalRepo, err) } } @@ -231,7 +231,7 @@ func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName string if cache.Address() == storage.LocalStorageAddress || cache.Address() == srcRepo { continue } - if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy SBOM artifacts into cache %s", werfImgName, cache.String()).DoError(func() error { + if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy attached artifacts into cache %s", werfImgName, cache.String()).DoError(func() error { return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, cache.Address(), srcDigest) }); err != nil { logboek.Context(ctx).Warn().LogF("Warning: unable to copy attached artifacts into cache stages storage %s: %s\n", cache.String(), err) diff --git a/pkg/build/sbom_step_propagate_test.go b/pkg/build/sbom_step_propagate_test.go index 0004e1cfef..ea4dc0be66 100644 --- a/pkg/build/sbom_step_propagate_test.go +++ b/pkg/build/sbom_step_propagate_test.go @@ -15,7 +15,6 @@ import ( "github.com/werf/werf/v3/pkg/attestation" "github.com/werf/werf/v3/pkg/docker_registry" - werfImage "github.com/werf/werf/v3/pkg/image" "github.com/werf/werf/v3/pkg/oci/artifact" "github.com/werf/werf/v3/pkg/storage" "github.com/werf/werf/v3/test/mock" @@ -55,16 +54,6 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) } - stageDescFor := func(repo, digest string) *werfImage.StageDesc { - return &werfImage.StageDesc{ - StageID: &werfImage.StageID{}, - Info: &werfImage.Info{ - Repository: repo, - RepoDigest: repo + "@" + digest, - }, - } - } - cacheStorage := func(address string) storage.StagesStorage { s := mock.NewMockStagesStorage(gomock.NewController(GinkgoT())) s.EXPECT().Address().Return(address).AnyTimes() @@ -96,7 +85,7 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { copyImageByDigest(ctx, srcRepo, finalRepo, srcDigest) step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), stageDescFor(finalRepo, srcDigest), nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, finalRepo, srcDigest, nil)).To(Succeed()) finalStore := artifact.NewOCIStore(finalRepo, "app", remoteOpts...) content, err := finalStore.GetAttachedContent(ctx, srcDigest, attestation.DSSEMediaType, nil) @@ -113,7 +102,7 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { cacheStorage(srcRepo), cacheStorage(cacheRepo), } - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), nil, caches)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "", "", caches)).To(Succeed()) cacheStore := artifact.NewOCIStore(cacheRepo, "app", remoteOpts...) content, err := cacheStore.GetAttachedContent(ctx, srcDigest, attestation.DSSEMediaType, nil) @@ -123,23 +112,23 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { It("should do nothing without a final repo and caches", func(ctx SpecContext) { step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), nil, nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "", "", nil)).To(Succeed()) }) It("should skip the final repo when it matches the stages repo", func(ctx SpecContext) { step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), stageDescFor(srcRepo, srcDigest), nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, srcRepo, srcDigest, nil)).To(Succeed()) }) It("should not fail when a cache repo is unreachable", func(ctx SpecContext) { step := &sbomStep{} caches := []storage.StagesStorage{cacheStorage("127.0.0.1:1/unreachable/cache")} - Expect(step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), nil, caches)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "", "", caches)).To(Succeed()) }) It("should fail when the final repo copy fails", func(ctx SpecContext) { step := &sbomStep{} - err := step.PropagateArtifacts(ctx, "app", stageDescFor(srcRepo, srcDigest), stageDescFor("127.0.0.1:1/unreachable/final", srcDigest), nil) + err := step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "127.0.0.1:1/unreachable/final", srcDigest, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("copy attached artifacts into final repo")) }) diff --git a/pkg/build/stages/remote_storage.go b/pkg/build/stages/remote_storage.go index 9161beabb6..6fab8a4dc4 100644 --- a/pkg/build/stages/remote_storage.go +++ b/pkg/build/stages/remote_storage.go @@ -8,6 +8,7 @@ import ( "github.com/werf/werf/v3/pkg/build" "github.com/werf/werf/v3/pkg/docker_registry" "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/oci/artifact" "github.com/werf/werf/v3/pkg/ref" "github.com/werf/werf/v3/pkg/storage/manager" ) @@ -83,6 +84,12 @@ func (s *RemoteStorage) copyCurrentBuildStagesFromRemote(ctx context.Context, fr if err = fromRemote.RegistryClient.CopyImage(ctx, infoGetterName, reference.FullName(), docker_registry.CopyImageOptions{}); err != nil { return fmt.Errorf("error copying stage %s into %s: %w", infoGetterName, reference.FullName(), err) } + + if infoGetter.Digest != "" { + if err := artifact.CopyAllAttachedArtifacts(ctx, infoGetter.Repo, infoGetter.Digest, reference.Repo, infoGetter.Digest); err != nil { + return fmt.Errorf("error copying artifacts attached to stage %s into %s: %w", infoGetterName, reference.Repo, err) + } + } } return nil @@ -116,6 +123,10 @@ func (s *RemoteStorage) copyAllFromRemote(ctx context.Context, fromRemote *Remot if err = fromRemote.RegistryClient.CopyImage(ctx, stageName, reference.FullName(), docker_registry.CopyImageOptions{}); err != nil { return fmt.Errorf("error copying stage %s into %s: %w", stageName, reference.FullName(), err) } + + if err := artifact.CopyAllAttachedArtifacts(ctx, stageDesc.Info.Repository, stageDesc.Info.GetDigest(), reference.Repo, stageDesc.Info.GetDigest()); err != nil { + return fmt.Errorf("error copying artifacts attached to stage %s into %s: %w", stageName, reference.Repo, err) + } } return nil diff --git a/pkg/deploy/bundles/copy_test.go b/pkg/deploy/bundles/copy_test.go index 18dc000f57..6cc8c4df48 100644 --- a/pkg/deploy/bundles/copy_test.go +++ b/pkg/deploy/bundles/copy_test.go @@ -15,6 +15,7 @@ import ( chartcommon "github.com/werf/nelm/v2/pkg/helm/pkg/chart/common" chart "github.com/werf/nelm/v2/pkg/helm/pkg/chart/v2" "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" "github.com/werf/werf/v3/pkg/logging" bundles_registry "github.com/werf/werf/v3/pkg/ref" ) @@ -686,6 +687,13 @@ func (registry *DockerRegistryStub) PullImageArchive(ctx context.Context, archiv return nil } +// TryGetRepoImage reports every image as absent: the stub holds archives, not +// registry manifests, so there are no attached artifacts to resolve and the +// artifact-carrying step is skipped. +func (registry *DockerRegistryStub) TryGetRepoImage(_ context.Context, _ string) (*image.Info, error) { + return nil, nil +} + func (registry *DockerRegistryStub) CopyImage(_ context.Context, sourceReference, destinationReference string, _ docker_registry.CopyImageOptions) error { data, hasImage := registry.ImagesByReference[sourceReference] if !hasImage { diff --git a/pkg/deploy/bundles/remote_bundle.go b/pkg/deploy/bundles/remote_bundle.go index 085069f023..1f31b7ccd2 100644 --- a/pkg/deploy/bundles/remote_bundle.go +++ b/pkg/deploy/bundles/remote_bundle.go @@ -13,6 +13,7 @@ import ( nelmcommon "github.com/werf/nelm/v2/pkg/common" chart "github.com/werf/nelm/v2/pkg/helm/pkg/chart/v2" "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/oci/artifact" bundles_registry "github.com/werf/werf/v3/pkg/ref" ) @@ -182,6 +183,7 @@ func (bundle *RemoteBundle) CopyFromRemote(ctx context.Context, fromRemote *Remo return err } + srcRepo := ref.Repo ref.Repo = bundle.RegistryAddress.Repo // TODO: copy images in parallel @@ -192,6 +194,14 @@ func (bundle *RemoteBundle) CopyFromRemote(ctx context.Context, fromRemote *Remo if err := fromRemote.RegistryClient.CopyImage(ctx, image, ref.FullName(), docker_registry.CopyImageOptions{}); err != nil { return fmt.Errorf("error copying image %s into %s: %w", image, ref.FullName(), err) } + + if imgInfo, err := fromRemote.RegistryClient.TryGetRepoImage(ctx, image); err != nil { + return fmt.Errorf("error resolving image %s to copy its attached artifacts: %w", image, err) + } else if imgInfo != nil { + if err := artifact.CopyAllAttachedArtifacts(ctx, srcRepo, imgInfo.GetDigest(), ref.Repo, imgInfo.GetDigest()); err != nil { + return fmt.Errorf("error copying artifacts attached to image %s into %s: %w", image, ref.Repo, err) + } + } } newImageVals[imageName] = ref.FullName() diff --git a/pkg/oci/artifact/copy.go b/pkg/oci/artifact/copy.go index 70a2da9ffb..788e1047cc 100644 --- a/pkg/oci/artifact/copy.go +++ b/pkg/oci/artifact/copy.go @@ -12,6 +12,34 @@ import ( "github.com/werf/werf/v3/pkg/image" ) +// CopyAllAttachedArtifacts copies every artifact attached to srcDigest in srcRepo +// onto dstDigest in dstRepo, and when srcDigest is an image index it also copies +// the artifacts attached to every manifest the index references. Referenced +// manifests are addressed by the same digest in both repositories, which holds for +// registry-level copies — the only in-scope way an index travels between +// repositories. +func CopyAllAttachedArtifacts(ctx context.Context, srcRepo, srcDigest, dstRepo, dstDigest string, opts ...remote.Option) error { + if err := CopyAttachedArtifacts(ctx, srcRepo, srcDigest, dstRepo, dstDigest, opts...); err != nil { + return err + } + + entries, err := ListIndexPlatforms(ctx, srcRepo, srcDigest, opts...) + if err != nil { + return fmt.Errorf("list index manifests of %s: %w", srcRepo+"@"+srcDigest, err) + } + + for _, entry := range entries { + if entry.Digest == srcDigest { + continue + } + if err := CopyAttachedArtifacts(ctx, srcRepo, entry.Digest, dstRepo, entry.Digest, opts...); err != nil { + return fmt.Errorf("copy artifacts of index manifest %s: %w", entry.Digest, err) + } + } + + return nil +} + // CopyAttachedArtifacts copies every artifact attached to srcDigest in srcRepo onto // dstDigest in dstRepo. Artifacts are re-attached from their payload rather than // copied manifest-by-manifest, so the parent digests may differ (e.g. when a stage diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index d537e1dd51..4d5a9b3ae5 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -825,8 +825,19 @@ func (m *StorageManager) CopySuitableStageDescByDigest(ctx context.Context, stag return nil, fmt.Errorf("unable to get stage %s description from %s: %w", stageDesc.StageID.String(), destinationStagesStorage.String(), err) } else { if sourceStagesStorage.Address() != storage.LocalStorageAddress && destinationStagesStorage.Address() != storage.LocalStorageAddress { - if err := artifact.CopyAttachedArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest(), destinationStagesStorage.Address(), destinationStageDesc.Info.GetDigest()); err != nil { - return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageDesc.StageID.String(), err) + // The backend-mediated copy does not guarantee digest preservation. When the + // digest survived, the attached artifacts still describe the copied image and + // are carried byte-identical. When it changed, a statement about the source + // digest is not a statement about the destination digest: werf-generated + // artifacts are regenerated by convergence against the new digest in the same + // run, and artifacts werf cannot regenerate are left behind. + if destinationStageDesc.Info.GetDigest() == stageDesc.Info.GetDigest() { + if err := artifact.CopyAllAttachedArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest(), destinationStagesStorage.Address(), destinationStageDesc.Info.GetDigest()); err != nil { + return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageDesc.StageID.String(), err) + } + } else { + logboek.Context(ctx).Warn().LogF("WARNING: artifacts attached to stage %s in %s are not carried over: the copy into %s changed the image digest (%s -> %s), so werf-generated artifacts will be regenerated and any user-signed attestations have to be re-issued against the new digest\n", + stageDesc.StageID.String(), sourceStagesStorage.String(), destinationStagesStorage.String(), stageDesc.Info.GetDigest(), destinationStageDesc.Info.GetDigest()) } } return destinationStageDesc, nil diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 1b2f1d33dc..f8ad696023 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -946,6 +946,13 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } if desc != nil { + // The manifest is already in place, but its artifacts may not be: an earlier + // run could have copied the stage and failed before the artifacts, or the + // artifacts could have appeared in the source afterwards. The copy is + // idempotent, so repeating it here repairs such a destination. + if err := artifact.CopyAllAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { + return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageID, err) + } return desc, nil } @@ -960,7 +967,7 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } - if err := artifact.CopyAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { + if err := artifact.CopyAllAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageID, err) } From 3d9d5d511a92a40a1d233e0bbc3f58c45bcc73cf Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 02:08:45 +0300 Subject: [PATCH 05/11] fix(build, sbom): stop artifact propagation from failing builds and losing compose refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Artifact propagation reached for the source manifest of every stage it touched even when the source did not hold it, and a 404 there aborted the whole copy — including the repair pass that now runs whenever a destination already holds a stage. Treat a missing source manifest like a missing artifact index: nothing to carry. Skip propagation entirely on local stages storage, and stop offering the index digest of a multi-platform image to cache repos, which never hold it. Route werf compose and the deploy values through the published descriptor, so a build with --final-repo hands out the final repo reference again instead of the stages repo one, matching what the build report and the --use-build-report path already carry. On the secondary-to-primary copy, warn only about artifacts a build cannot regenerate — the SBOM and the VEX document come back through convergence, an attestation signed by the user does not. Signed-off-by: Radmir Khurum --- pkg/build/build_phase.go | 57 ++++++++++++-------------- pkg/build/conveyor.go | 7 +++- pkg/build/sbom_step.go | 21 +++++++--- pkg/build/sbom_step_propagate_test.go | 12 +++--- pkg/build/stages/remote_storage.go | 6 ++- pkg/oci/artifact/copy.go | 10 +++++ pkg/oci/artifact/fallback.go | 32 +++++++++++++++ pkg/storage/manager/storage_manager.go | 12 ++++-- pkg/storage/repo_stages_storage.go | 5 ++- 9 files changed, 112 insertions(+), 50 deletions(-) diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 2cde9efa23..5ae555c576 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -689,44 +689,44 @@ func (phase *BuildPhase) convergePlatformImageSbom(ctx context.Context, name str // so a destination holding the image without its artifacts is repaired by the next // run. For a multi-platform image the per-platform artifacts are copied onto the // platform manifest digests — preserved by the registry-level index copy — and the -// image-level artifacts (e.g. VEX) onto the index digest. +// image-level artifacts (e.g. VEX) onto the index digest, which exists in the primary +// stages storage and in the final repo but never in a cache repo. func (phase *BuildPhase) propagateArtifacts(ctx context.Context) error { + if _, isLocal := phase.Conveyor.StorageManager.GetStagesStorage().(*storage.LocalStagesStorage); isLocal { + return nil + } + cacheStagesStorageList := phase.Conveyor.StorageManager.GetCacheStagesStorageList() for _, pair := range phase.Conveyor.imagesTree.GetImagesByName(false) { name, images := pair.Unpair() if multiImg := phase.Conveyor.imagesTree.GetMultiplatformImage(name); multiImg != nil { - var finalRepo string finalDesc := multiImg.GetFinalStageDesc() - if finalDesc != nil { - finalRepo = finalDesc.Info.Repository - } for _, img := range images { stageDesc := img.GetContentTagDesc() if stageDesc == nil { continue } - if err := phase.sbomStep.PropagateArtifacts(ctx, name, - stageDesc.Info.Repository, stageDesc.Info.GetDigest(), - finalRepo, stageDesc.Info.GetDigest(), - cacheStagesStorageList, - ); err != nil { + + opts := PropagateArtifactsOptions{CacheStagesStorageList: cacheStagesStorageList} + if finalDesc != nil { + opts.FinalRepo = finalDesc.Info.Repository + opts.FinalDigest = stageDesc.Info.GetDigest() + } + + if err := phase.sbomStep.PropagateArtifacts(ctx, name, stageDesc.Info.Repository, stageDesc.Info.GetDigest(), opts); err != nil { return fmt.Errorf("unable to propagate artifacts for image %q (platform %s): %w", name, img.TargetPlatform, err) } } - if idxDesc := multiImg.GetStageDesc(); idxDesc != nil { - var finalDigest string - if finalDesc != nil { - finalDigest = finalDesc.Info.GetDigest() + if idxDesc := multiImg.GetStageDesc(); idxDesc != nil && finalDesc != nil { + opts := PropagateArtifactsOptions{ + FinalRepo: finalDesc.Info.Repository, + FinalDigest: finalDesc.Info.GetDigest(), } - if err := phase.sbomStep.PropagateArtifacts(ctx, name, - idxDesc.Info.Repository, idxDesc.Info.GetDigest(), - finalRepo, finalDigest, - cacheStagesStorageList, - ); err != nil { + if err := phase.sbomStep.PropagateArtifacts(ctx, name, idxDesc.Info.Repository, idxDesc.Info.GetDigest(), opts); err != nil { return fmt.Errorf("unable to propagate artifacts for image %q: %w", name, err) } } @@ -734,23 +734,20 @@ func (phase *BuildPhase) propagateArtifacts(ctx context.Context) error { continue } - img := images[0] - stageDesc := img.GetContentTagDesc() + // An image built out of no stages has no descriptor and nothing attached to + // propagate. + stageDesc := images[0].GetContentTagDesc() if stageDesc == nil { continue } - var finalRepo, finalDigest string - if finalDesc := img.GetFinalContentTagDesc(); finalDesc != nil { - finalRepo = finalDesc.Info.Repository - finalDigest = finalDesc.Info.GetDigest() + opts := PropagateArtifactsOptions{CacheStagesStorageList: cacheStagesStorageList} + if finalDesc := images[0].GetFinalContentTagDesc(); finalDesc != nil { + opts.FinalRepo = finalDesc.Info.Repository + opts.FinalDigest = finalDesc.Info.GetDigest() } - if err := phase.sbomStep.PropagateArtifacts(ctx, name, - stageDesc.Info.Repository, stageDesc.Info.GetDigest(), - finalRepo, finalDigest, - cacheStagesStorageList, - ); err != nil { + if err := phase.sbomStep.PropagateArtifacts(ctx, name, stageDesc.Info.Repository, stageDesc.Info.GetDigest(), opts); err != nil { return fmt.Errorf("unable to propagate artifacts for image %q: %w", name, err) } } diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index c5f4db2339..24b6eb5fb7 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -539,7 +539,7 @@ func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) ([]*imag if len(platforms) == 1 { img := images[0] - getter := c.StorageManager.GetImageInfoGetter(img.Name, img.GetContentTagDesc(), opts) + getter := c.StorageManager.GetImageInfoGetter(img.Name, img.GetPublishedContentTagDesc(), opts) imagesGetters = append(imagesGetters, getter) } else { img := c.imagesTree.GetMultiplatformImage(name) @@ -581,7 +581,10 @@ func (c *Conveyor) GetImagesEnvArray() []string { continue } - envArray = append(envArray, GenerateImageEnv(img.Name, c.GetImageContentTagName(img.TargetPlatform, img.Name))) + // werf compose hands these references to docker compose, so they have to name + // the image where it was published — the final repo when the build used one, + // matching what GetImagesEnvArrayFromReport reads out of the build report. + envArray = append(envArray, GenerateImageEnv(img.Name, img.GetPublishedContentTagDesc().Info.Name)) } return envArray diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index 574a91dc41..d6c84509d3 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -212,22 +212,31 @@ func (step *sbomStep) calculateStableChecksum(scanOpts scanner.ScanOptions, merg ) } +// PropagateArtifactsOptions carries the destinations of a propagation: the final repo +// descriptor of the same image when the build published one, and the cache stages +// storages the stage was placed in. +type PropagateArtifactsOptions struct { + FinalRepo string + FinalDigest string + CacheStagesStorageList []storage.StagesStorage +} + // PropagateArtifacts copies the artifacts attached to the image in the repository it // was built in — its SBOM, VEX and any other attached kind — into the final repo and // the cache repos. Stages are copied there before the artifacts exist, so the // artifacts have to catch up separately. The copy runs on every build and is // idempotent, so a destination holding the image without its artifacts is repaired // by the next run. -func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName, srcRepo, srcDigest, finalRepo, finalDigest string, cacheStagesStorageList []storage.StagesStorage) error { - if finalRepo != "" && finalRepo != srcRepo { - if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy attached artifacts into the final repo %s", werfImgName, finalRepo).DoError(func() error { - return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, finalRepo, finalDigest) +func (step *sbomStep) PropagateArtifacts(ctx context.Context, werfImgName, srcRepo, srcDigest string, opts PropagateArtifactsOptions) error { + if opts.FinalRepo != "" && opts.FinalRepo != srcRepo { + if err := logboek.Context(ctx).Info().LogProcess("image %s: Copy attached artifacts into the final repo %s", werfImgName, opts.FinalRepo).DoError(func() error { + return artifact.CopyAttachedArtifacts(ctx, srcRepo, srcDigest, opts.FinalRepo, opts.FinalDigest) }); err != nil { - return fmt.Errorf("copy attached artifacts into final repo %s: %w", finalRepo, err) + return fmt.Errorf("copy attached artifacts into final repo %s: %w", opts.FinalRepo, err) } } - for _, cache := range cacheStagesStorageList { + for _, cache := range opts.CacheStagesStorageList { if cache.Address() == storage.LocalStorageAddress || cache.Address() == srcRepo { continue } diff --git a/pkg/build/sbom_step_propagate_test.go b/pkg/build/sbom_step_propagate_test.go index ea4dc0be66..3c4726b3f0 100644 --- a/pkg/build/sbom_step_propagate_test.go +++ b/pkg/build/sbom_step_propagate_test.go @@ -85,7 +85,7 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { copyImageByDigest(ctx, srcRepo, finalRepo, srcDigest) step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, finalRepo, srcDigest, nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{FinalRepo: finalRepo, FinalDigest: srcDigest})).To(Succeed()) finalStore := artifact.NewOCIStore(finalRepo, "app", remoteOpts...) content, err := finalStore.GetAttachedContent(ctx, srcDigest, attestation.DSSEMediaType, nil) @@ -102,7 +102,7 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { cacheStorage(srcRepo), cacheStorage(cacheRepo), } - Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "", "", caches)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{CacheStagesStorageList: caches})).To(Succeed()) cacheStore := artifact.NewOCIStore(cacheRepo, "app", remoteOpts...) content, err := cacheStore.GetAttachedContent(ctx, srcDigest, attestation.DSSEMediaType, nil) @@ -112,23 +112,23 @@ var _ = Describe("SbomStep PropagateArtifacts", func() { It("should do nothing without a final repo and caches", func(ctx SpecContext) { step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "", "", nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{})).To(Succeed()) }) It("should skip the final repo when it matches the stages repo", func(ctx SpecContext) { step := &sbomStep{} - Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, srcRepo, srcDigest, nil)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{FinalRepo: srcRepo, FinalDigest: srcDigest})).To(Succeed()) }) It("should not fail when a cache repo is unreachable", func(ctx SpecContext) { step := &sbomStep{} caches := []storage.StagesStorage{cacheStorage("127.0.0.1:1/unreachable/cache")} - Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "", "", caches)).To(Succeed()) + Expect(step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{CacheStagesStorageList: caches})).To(Succeed()) }) It("should fail when the final repo copy fails", func(ctx SpecContext) { step := &sbomStep{} - err := step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, "127.0.0.1:1/unreachable/final", srcDigest, nil) + err := step.PropagateArtifacts(ctx, "app", srcRepo, srcDigest, PropagateArtifactsOptions{FinalRepo: "127.0.0.1:1/unreachable/final", FinalDigest: srcDigest}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("copy attached artifacts into final repo")) }) diff --git a/pkg/build/stages/remote_storage.go b/pkg/build/stages/remote_storage.go index 6fab8a4dc4..221d5cee19 100644 --- a/pkg/build/stages/remote_storage.go +++ b/pkg/build/stages/remote_storage.go @@ -124,8 +124,10 @@ func (s *RemoteStorage) copyAllFromRemote(ctx context.Context, fromRemote *Remot return fmt.Errorf("error copying stage %s into %s: %w", stageName, reference.FullName(), err) } - if err := artifact.CopyAllAttachedArtifacts(ctx, stageDesc.Info.Repository, stageDesc.Info.GetDigest(), reference.Repo, stageDesc.Info.GetDigest()); err != nil { - return fmt.Errorf("error copying artifacts attached to stage %s into %s: %w", stageName, reference.Repo, err) + if digest := stageDesc.Info.GetDigest(); digest != "" { + if err := artifact.CopyAllAttachedArtifacts(ctx, stageDesc.Info.Repository, digest, reference.Repo, digest); err != nil { + return fmt.Errorf("error copying artifacts attached to stage %s into %s: %w", stageName, reference.Repo, err) + } } } diff --git a/pkg/oci/artifact/copy.go b/pkg/oci/artifact/copy.go index 788e1047cc..161d2b64bd 100644 --- a/pkg/oci/artifact/copy.go +++ b/pkg/oci/artifact/copy.go @@ -2,10 +2,12 @@ package artifact import ( "context" + "errors" "fmt" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/werf/logboek" "github.com/werf/werf/v3/pkg/docker_registry" @@ -25,6 +27,14 @@ func CopyAllAttachedArtifacts(ctx context.Context, srcRepo, srcDigest, dstRepo, entries, err := ListIndexPlatforms(ctx, srcRepo, srcDigest, opts...) if err != nil { + // A source that does not hold the manifest holds no artifacts attached to it + // either, which is the same no-op CopyAttachedArtifacts makes of a missing + // fallback index rather than a reason to fail the operation that copied the + // image. + var transportErr *transport.Error + if errors.As(err, &transportErr) && transportErr.StatusCode == 404 { + return nil + } return fmt.Errorf("list index manifests of %s: %w", srcRepo+"@"+srcDigest, err) } diff --git a/pkg/oci/artifact/fallback.go b/pkg/oci/artifact/fallback.go index 77e586bc19..e9d6e31d4e 100644 --- a/pkg/oci/artifact/fallback.go +++ b/pkg/oci/artifact/fallback.go @@ -19,6 +19,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/samber/lo" "github.com/werf/logboek" "github.com/werf/werf/v3/pkg/image" @@ -304,6 +305,37 @@ func GetAttached(ctx context.Context, repo, parentDigest, artifactType, imageNam return matches[0], true, nil } +// ListUnregenerableArtifacts returns the predicate types of the artifacts attached to +// parentDigest that a build cannot produce again. Everything werf generates itself — +// the SBOM and the VEX document — carries the checksum annotation of its cache +// identity, while an attestation signed through werf attest sign carries none: its +// predicate and its signing key exist only on the user's side. +func ListUnregenerableArtifacts(ctx context.Context, repo, parentDigest string, opts ...remote.Option) ([]string, error) { + idx, err := pullFallbackIndex(ctx, repo, parentDigest, opts...) + if err != nil { + return nil, err + } + + im, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("read fallback index manifest: %w", err) + } + + var types []string + for _, desc := range im.Manifests { + if desc.ArtifactType == "" || desc.Annotations[image.WerfChecksumAnnotation] != "" { + continue + } + predicateType := desc.Annotations[PredicateTypeAnnotation] + if predicateType == "" { + predicateType = desc.ArtifactType + } + types = append(types, predicateType) + } + + return lo.Uniq(types), nil +} + func multipleArtifactEntriesWarning(parentDigest string, matches []v1.Descriptor) string { names := make([]string, 0, len(matches)) for _, desc := range matches { diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index 4d5a9b3ae5..be6c331206 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -830,14 +830,20 @@ func (m *StorageManager) CopySuitableStageDescByDigest(ctx context.Context, stag // are carried byte-identical. When it changed, a statement about the source // digest is not a statement about the destination digest: werf-generated // artifacts are regenerated by convergence against the new digest in the same - // run, and artifacts werf cannot regenerate are left behind. + // run, and only artifacts werf cannot regenerate are worth reporting. if destinationStageDesc.Info.GetDigest() == stageDesc.Info.GetDigest() { if err := artifact.CopyAllAttachedArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest(), destinationStagesStorage.Address(), destinationStageDesc.Info.GetDigest()); err != nil { return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageDesc.StageID.String(), err) } } else { - logboek.Context(ctx).Warn().LogF("WARNING: artifacts attached to stage %s in %s are not carried over: the copy into %s changed the image digest (%s -> %s), so werf-generated artifacts will be regenerated and any user-signed attestations have to be re-issued against the new digest\n", - stageDesc.StageID.String(), sourceStagesStorage.String(), destinationStagesStorage.String(), stageDesc.Info.GetDigest(), destinationStageDesc.Info.GetDigest()) + leftBehind, err := artifact.ListUnregenerableArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest()) + if err != nil { + return nil, fmt.Errorf("unable to list artifacts attached to stage %s in %s: %w", stageDesc.StageID.String(), sourceStagesStorage.String(), err) + } + if len(leftBehind) > 0 { + logboek.Context(ctx).Warn().LogF("WARNING: attestations [%s] of stage %s stay in %s: the copy changed the image digest, re-issue them against %s\n", + strings.Join(leftBehind, ", "), stageDesc.StageID.String(), sourceStagesStorage.String(), destinationStageDesc.Info.GetDigest()) + } } } return destinationStageDesc, nil diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index f8ad696023..955cbaa8e1 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -949,7 +949,10 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage // The manifest is already in place, but its artifacts may not be: an earlier // run could have copied the stage and failed before the artifacts, or the // artifacts could have appeared in the source afterwards. The copy is - // idempotent, so repeating it here repairs such a destination. + // idempotent, so repeating it here repairs such a destination. Both + // repositories are addressed by the same digest because a stage reaches this + // destination through a registry-level copy, which preserves it; a source that + // does not hold that digest is a no-op rather than a failure. if err := artifact.CopyAllAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageID, err) } From fcbd1c7390e3564b074c66834ac1c691addb7ca2 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 02:11:36 +0300 Subject: [PATCH 06/11] test(sbom): rebuild the final repo specs on the current suite API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The specs were written against the suite API that #348 removed: a setupSbomBuildEnv taking backend options, a per-backend entry table, and a multiplatform fixture that no longer exists. All three are gone — the entries differed only in DOCKER_BUILDKIT while every fixture is stapel, so both ran the same code path — and the multi-platform spec brings its own fixture. Strengthen what the specs prove. The dependent specs now check the artifact in the final repo names the image and carries an in-toto subject equal to the digest it hangs on, so the SBOM of the base image no longer satisfies them, and they read the same SBOM back out of the stages repo. The multi-platform spec checks the same subject per platform manifest. The cleanup spec pins the stage with a keep list instead of git reachability, so the same run deletes every other stage, and then asserts the orphaned artifacts are collected once the image itself is gone. Signed-off-by: Radmir Khurum --- .../final_repo_multiplatform/werf.yaml | 15 ++ test/e2e/sbom/final_repo_cleanup_test.go | 166 ++++++++-------- test/e2e/sbom/final_repo_dependent_test.go | 177 ++++++++++-------- .../e2e/sbom/final_repo_multiplatform_test.go | 120 ++++++------ 4 files changed, 257 insertions(+), 221 deletions(-) create mode 100644 test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml diff --git a/test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml new file mode 100644 index 0000000000..4145936be1 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_multiplatform/werf.yaml @@ -0,0 +1,15 @@ +project: werf-test-e2e-sbom-final-repo-multiplatform +configVersion: 1 +build: + platform: + - linux/amd64 + - linux/arm64 + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: app +from: registry.werf.io/werf/scratch:latest +git: + - add: / + to: /app diff --git a/test/e2e/sbom/final_repo_cleanup_test.go b/test/e2e/sbom/final_repo_cleanup_test.go index 2636de0bd5..1c7677c40f 100644 --- a/test/e2e/sbom/final_repo_cleanup_test.go +++ b/test/e2e/sbom/final_repo_cleanup_test.go @@ -1,102 +1,96 @@ package e2e_build_test import ( + "os" + "path/filepath" "strings" + "github.com/google/go-containerregistry/pkg/crane" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/werf/werf/v3/test/pkg/report" sbomtest "github.com/werf/werf/v3/test/pkg/sbom" "github.com/werf/werf/v3/test/pkg/suite_init" - "github.com/werf/werf/v3/test/pkg/utils" "github.com/werf/werf/v3/test/pkg/werf" ) var _ = Describe("SBOM retention across cleanup", Label("e2e", "sbom", "final-repo", "cleanup"), func() { - DescribeTable("build with --final-repo → cleanup → SBOM still readable from both repos", - func(ctx SpecContext, testOpts sbomTestOptions) { - setupSbomBuildEnv(testOpts.setupEnvOptions) - - stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) - finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") - SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) - - repoDirname := "repo_sbom_final_repo_cleanup" - SuiteData.InitTestRepo(ctx, repoDirname, "inject/ospm_basic") - testRepoPath := SuiteData.GetTestRepoPath(repoDirname) - - // werf cleanup keeps an image only while the commit it was built from stays - // reachable from a remote branch. Without origin every image looks unreachable - // and gets deleted, which would hide the retention this test asserts. - // The bare remote lives outside the work tree: the fixture adds the whole - // project directory to the image, so an in-tree remote would break giterminism. - remotePath := SuiteData.GetTestRepoPath(repoDirname + "_remote.git") - utils.RunSucceedCommand(ctx, testRepoPath, "git", "init", "--bare", remotePath) - utils.RunSucceedCommand(ctx, testRepoPath, "git", "remote", "add", "origin", remotePath) - utils.RunSucceedCommand(ctx, testRepoPath, "git", "push", "--set-upstream", "origin", "HEAD:refs/heads/main") - - builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-cleanup-builder") - - werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) - reportProject := report.NewProjectWithReport(werfProject) - _, buildReport := reportProject.BuildWithReport(ctx, - SuiteData.GetBuildReportPath("sbom_final_repo_cleanup.json"), - &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, - ) - - appRecord, found := buildReport.Images["app"] - Expect(found).To(BeTrue(), "expected image %q in build report", "app") - Expect(appRecord.DockerRepo).To(Equal(finalRepo), - "expected build report to reference the final repo") - finalDigest := appRecord.DockerImageDigest - Expect(finalDigest).NotTo(BeEmpty()) - - // The stage copy into the final repo may change the manifest digest, so the - // stages repo has to be addressed by its own stage tag rather than by finalDigest. - stageTag := stageTagOf(ctx, werfProject, "app", builderEnv) - - assertSbomInFinalRepo := func() { - sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ - CommonOptions: werf.CommonOptions{ - ExtraArgs: []string{"--repo", finalRepo, "--digest", finalDigest}, - Envs: builderEnv, - }, - }) - sbomtest.AssertHasComponent(sbomtest.MustParseSBOMOutput(sbomOut), "curl", "8.12.1") - } - - assertSbomInStagesRepo := func() { - sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ - CommonOptions: werf.CommonOptions{ - ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTag}, - Envs: builderEnv, - }, - }) - sbomtest.AssertHasComponent(sbomtest.MustParseSBOMOutput(sbomOut), "curl", "8.12.1") - } - - By("reading the SBOM from both repos before cleanup") - assertSbomInFinalRepo() - assertSbomInStagesRepo() - - By("running cleanup while the image is still in use") - werfProject.RunCommand(ctx, []string{"cleanup", "--without-kube"}, werf.CommonOptions{Envs: builderEnv}) - - By("reading the SBOM from both repos after cleanup") - assertSbomInFinalRepo() - assertSbomInStagesRepo() - - By("running cleanup a second time") - werfProject.RunCommand(ctx, []string{"cleanup", "--without-kube"}, werf.CommonOptions{Envs: builderEnv}) - - By("reading the SBOM from both repos after the repeated cleanup") - assertSbomInFinalRepo() - assertSbomInStagesRepo() - }, - Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), - Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), - ) + It("cleanup keeps the SBOM of a retained image in both repos and collects it once the image is deleted", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_cleanup" + SuiteData.InitTestRepo(ctx, repoDirname, "inject/ospm_basic") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-cleanup-builder") + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_cleanup.json"), + &werf.WithReportOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}, + ) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo), + "expected build report to reference the final repo") + finalDigest := appRecord.DockerImageDigest + Expect(finalDigest).NotTo(BeEmpty()) + + stageTag := stageTagOf(ctx, werfProject, "app", builderEnv) + + assertSbomReadable := func(repo string, extraArgs ...string) { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: append([]string{"--repo", repo}, extraArgs...), + Envs: builderEnv, + }, + }) + sbomtest.AssertHasComponent(sbomtest.MustParseSBOMOutput(sbomOut), "curl", "8.12.1") + } + + By("reading the SBOM from both repos before cleanup") + assertSbomReadable(finalRepo, "--digest", finalDigest) + assertSbomReadable(stagesRepo, "--tag", stageTag) + + // The keep list pins the stage without relying on git history reachability, + // so the retention this spec asserts cannot pass because nothing was + // eligible for deletion in the first place: the same run deletes every stage + // that is not on the list. + keepListPath := filepath.Join(testRepoPath, ".werf-keep-list") + Expect(os.WriteFile(keepListPath, []byte(stageTag+"\n"), 0o600)).To(Succeed()) + + cleanupArgs := []string{"cleanup", "--without-kube", "--keep-stages-built-within-last-n-hours=0"} + + By("running cleanup twice while the stage is on the keep list") + for range 2 { + werfProject.RunCommand(ctx, append(append([]string{}, cleanupArgs...), "--keep-list", keepListPath), + werf.CommonOptions{Envs: builderEnv}) + + assertSbomReadable(finalRepo, "--digest", finalDigest) + assertSbomReadable(stagesRepo, "--tag", stageTag) + } + + By("running cleanup with nothing protecting the stage") + werfProject.RunCommand(ctx, cleanupArgs, werf.CommonOptions{Envs: builderEnv}) + + registryOptions := []crane.Option{crane.Insecure, crane.WithContext(ctx)} + for _, repo := range []string{stagesRepo, finalRepo} { + tags, err := crane.ListTags(repo, registryOptions...) + Expect(err).NotTo(HaveOccurred()) + Expect(tags).NotTo(ContainElement(stageTag), "cleanup must have deleted the stage from %s", repo) + } + + By("checking the orphaned SBOM artifacts were collected in both repos") + expectNoSbomArtifact(ctx, stagesRepo, finalDigest) + expectNoSbomArtifact(ctx, finalRepo, finalDigest) + }) }) // stageTagOf returns the content-based tag of the image's last stage in the stages repo. @@ -108,9 +102,9 @@ func stageTagOf(ctx SpecContext, werfProject *werf.Project, imageName string, en ref := strings.TrimSpace(lines[len(lines)-1]) Expect(ref).NotTo(BeEmpty(), "expected a stage image reference for %q, got output:\n%s", imageName, out) - parts := strings.Split(ref, ":") - tag := parts[len(parts)-1] + tag := ref[strings.LastIndex(ref, ":")+1:] Expect(tag).NotTo(BeEmpty(), "expected a tag in the stage image reference %q", ref) + Expect(tag).NotTo(ContainSubstring("@"), "expected a tagged stage image reference, got %q", ref) return tag } diff --git a/test/e2e/sbom/final_repo_dependent_test.go b/test/e2e/sbom/final_repo_dependent_test.go index 96cca8f3ab..b31f6e39bd 100644 --- a/test/e2e/sbom/final_repo_dependent_test.go +++ b/test/e2e/sbom/final_repo_dependent_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/image" "github.com/werf/werf/v3/test/pkg/report" sbomtest "github.com/werf/werf/v3/test/pkg/sbom" "github.com/werf/werf/v3/test/pkg/suite_init" @@ -16,84 +18,105 @@ import ( // builder base inherits the io.deckhouse.internal.builder label, and together // with WERF_E2E_ALLOW_LOCAL_BUILDER_IMAGES a failed base SBOM lookup silently // degrades into ErrSbomNotRequired instead of failing the build. A scratch -// base leaves no such escape, so these tables actually falsify the base/import +// base leaves no such escape, so these specs actually falsify the base/import // SBOM lookup. var _ = Describe("SBOM final repo with dependent images", Label("e2e", "sbom", "final-repo", "dependent"), func() { - DescribeTable("image built from another image of the project: build with --final-repo succeeds", - func(ctx SpecContext, testOpts sbomTestOptions) { - setupSbomBuildEnv(testOpts.setupEnvOptions) - - finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") - SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) - - repoDirname := "repo_sbom_final_repo_dependent" - SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_dependent") - testRepoPath := SuiteData.GetTestRepoPath(repoDirname) - - // The build itself is the primary assertion: SBOM convergence of the - // dependent image has to find the SBOM of its base image, so a lookup - // pointed at a repository that does not hold it fails the build. - By("building the two dependent images with --final-repo against a clean registry") - werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) - reportProject := report.NewProjectWithReport(werfProject) - _, buildReport := reportProject.BuildWithReport(ctx, - SuiteData.GetBuildReportPath("sbom_final_repo_dependent.json"), nil) - - appRecord, found := buildReport.Images["app"] - Expect(found).To(BeTrue(), "expected image %q in build report", "app") - Expect(appRecord.DockerRepo).To(Equal(finalRepo)) - Expect(appRecord.DockerImageDigest).NotTo(BeEmpty()) - - By("reading the dependent image's SBOM from the final repo") - sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ - CommonOptions: werf.CommonOptions{ - ExtraArgs: []string{"--repo", finalRepo, "--digest", appRecord.DockerImageDigest}, - }, - }) - sbomtest.MustParseSBOMOutput(sbomOut) - - By("rebuilding and checking the SBOMs are served from cache, not regenerated") - rebuildOut := werfProject.Build(ctx, nil) - Expect(strings.Count(rebuildOut, "Use previously generated SBOM from registry")).To(BeNumerically(">=", 2), - "both the base and the dependent image SBOMs must be reused on rebuild") + It("image built from another image of the project: build with --final-repo succeeds", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_dependent" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_dependent") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + // The build itself is the primary assertion: SBOM convergence of the + // dependent image has to find the SBOM of its base image, so a lookup + // pointed at a repository that does not hold it fails the build. + By("building the two dependent images with --final-repo against a clean registry") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_dependent.json"), nil) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + appDigest := appRecord.DockerImageDigest + Expect(appDigest).NotTo(BeEmpty()) + + assertAppSbomInFinalRepo(ctx, werfProject, finalRepo, appDigest) + assertAppSbomInStagesRepo(ctx, werfProject, stagesRepo, "app") + + By("rebuilding and checking the SBOMs are served from cache, not regenerated") + rebuildOut := werfProject.Build(ctx, nil) + Expect(strings.Count(rebuildOut, "Use previously generated SBOM from registry")).To(BeNumerically(">=", 2), + "both the base and the dependent image SBOMs must be reused on rebuild") + }) + + It("image importing files from another image of the project: build with --final-repo succeeds", func(ctx SpecContext) { + setupSbomBuildEnv() + + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + + repoDirname := "repo_sbom_final_repo_import" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_import") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + // SBOM convergence of the importing image has to find the SBOM of the + // import source, exercising the import-side lookup the same way the + // fromImage spec exercises the base-image one. + By("building the importing image with --final-repo against a clean registry") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_import.json"), nil) + + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + appDigest := appRecord.DockerImageDigest + Expect(appDigest).NotTo(BeEmpty()) + + assertAppSbomInFinalRepo(ctx, werfProject, finalRepo, appDigest) + assertAppSbomInStagesRepo(ctx, werfProject, stagesRepo, "app") + }) +}) + +// assertAppSbomInFinalRepo checks the final repo serves the SBOM of that very image: +// the artifact index entry has to name the image and its in-toto subject has to be +// the digest the artifact is attached to, so an SBOM of another image of the same +// project — the base image is the one at hand — does not satisfy the assertion. +func assertAppSbomInFinalRepo(ctx SpecContext, werfProject *werf.Project, finalRepo, digest string) { + By("reading the dependent image's SBOM from the final repo") + + desc, payload := fetchSingleSbomArtifact(ctx, finalRepo, digest) + Expect(desc.ArtifactType).To(Equal(attestation.DSSEMediaType)) + Expect(desc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + Expect(mustExtractInTotoSubjectDigest(payload)).To(Equal(digest)) + + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", finalRepo, "--digest", digest}, }, - Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), - Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), - ) - - DescribeTable("image importing files from another image of the project: build with --final-repo succeeds", - func(ctx SpecContext, testOpts sbomTestOptions) { - setupSbomBuildEnv(testOpts.setupEnvOptions) - - finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") - SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) - - repoDirname := "repo_sbom_final_repo_import" - SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_import") - testRepoPath := SuiteData.GetTestRepoPath(repoDirname) - - // SBOM convergence of the importing image has to find the SBOM of the - // import source, exercising the import-side lookup the same way the - // fromImage table exercises the base-image one. - By("building the importing image with --final-repo against a clean registry") - werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) - reportProject := report.NewProjectWithReport(werfProject) - _, buildReport := reportProject.BuildWithReport(ctx, - SuiteData.GetBuildReportPath("sbom_final_repo_import.json"), nil) - - appRecord, found := buildReport.Images["app"] - Expect(found).To(BeTrue(), "expected image %q in build report", "app") - Expect(appRecord.DockerImageDigest).NotTo(BeEmpty()) - - By("reading the importing image's SBOM from the final repo") - sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ - CommonOptions: werf.CommonOptions{ - ExtraArgs: []string{"--repo", finalRepo, "--digest", appRecord.DockerImageDigest}, - }, - }) - sbomtest.MustParseSBOMOutput(sbomOut) + }) + sbomtest.MustParseSBOMOutput(sbomOut) +} + +// assertAppSbomInStagesRepo reads the same SBOM out of the repository the image was +// built in, addressed by its stage tag: the presence of a copy in the final repo must +// not stand in for its absence here. +func assertAppSbomInStagesRepo(ctx SpecContext, werfProject *werf.Project, stagesRepo, imageName string) { + By("reading the same image's SBOM from the stages repo") + + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTagOf(ctx, werfProject, imageName, nil)}, }, - Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), - Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), - ) -}) + }) + sbomtest.MustParseSBOMOutput(sbomOut) +} diff --git a/test/e2e/sbom/final_repo_multiplatform_test.go b/test/e2e/sbom/final_repo_multiplatform_test.go index 84eec68be1..bb3e227ff8 100644 --- a/test/e2e/sbom/final_repo_multiplatform_test.go +++ b/test/e2e/sbom/final_repo_multiplatform_test.go @@ -4,6 +4,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/werf/werf/v3/pkg/attestation" "github.com/werf/werf/v3/pkg/image" "github.com/werf/werf/v3/test/pkg/report" sbomtest "github.com/werf/werf/v3/test/pkg/sbom" @@ -12,74 +13,77 @@ import ( ) var _ = Describe("SBOM final repo multi-platform", Label("e2e", "sbom", "final-repo", "multiplatform"), func() { - DescribeTable("build with --final-repo → per-platform SBOMs on platform manifest digests in both repos, none on the index", - func(ctx SpecContext, testOpts sbomTestOptions) { - setupSbomBuildEnv(testOpts.setupEnvOptions) + It("build with --final-repo → per-platform SBOMs on platform manifest digests in both repos, none on the index", func(ctx SpecContext) { + setupSbomBuildEnv() - stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) - finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") - SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) - SuiteData.Stubs.SetEnv("WERF_ENABLE_REPORT_BY_PLATFORM", "1") - SuiteData.Stubs.SetEnv("WERF_EXPERIMENTAL_STAPEL_ARM", "1") + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName) + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + SuiteData.Stubs.SetEnv("WERF_ENABLE_REPORT_BY_PLATFORM", "1") + SuiteData.Stubs.SetEnv("WERF_EXPERIMENTAL_STAPEL_ARM", "1") - repoDirname := "repo_sbom_final_repo_multiplatform" - SuiteData.InitTestRepo(ctx, repoDirname, "multiplatform") - testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + repoDirname := "repo_sbom_final_repo_multiplatform" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_multiplatform") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) - By("building the multi-platform image with --final-repo") - werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) - reportProject := report.NewProjectWithReport(werfProject) - _, buildReport := reportProject.BuildWithReport(ctx, - SuiteData.GetBuildReportPath("sbom_final_repo_multiplatform.json"), nil) + By("building the multi-platform image with --final-repo") + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_multiplatform.json"), nil) - appRecord, found := buildReport.Images["app"] - Expect(found).To(BeTrue(), "expected image %q in build report", "app") - Expect(appRecord.DockerRepo).To(Equal(finalRepo), - "expected build report to reference the final repo") - indexDigest := appRecord.DockerImageDigest - Expect(indexDigest).NotTo(BeEmpty()) + appRecord, found := buildReport.Images["app"] + Expect(found).To(BeTrue(), "expected image %q in build report", "app") + Expect(appRecord.DockerRepo).To(Equal(finalRepo), + "expected build report to reference the final repo") + indexDigest := appRecord.DockerImageDigest + Expect(indexDigest).NotTo(BeEmpty()) - byPlatform := buildReport.ImagesByPlatform["app"] - Expect(byPlatform).To(HaveLen(len(multiplatformSbomPlatforms)), "expected a build report record per platform") + byPlatform := buildReport.ImagesByPlatform["app"] + Expect(byPlatform).To(HaveLen(len(multiplatformSbomPlatforms)), "expected a build report record per platform") - // The registry-level index copy into the final repo preserves the digests - // of the platform manifests it references, so the same platform digest - // addresses the manifest in both repositories. - By("verifying each platform manifest carries its SBOM in both repositories") - for _, platform := range multiplatformSbomPlatforms { - record, hasRecord := byPlatform[platform] - Expect(hasRecord).To(BeTrue(), "no build report record for platform %s", platform) + // The registry-level index copy into the final repo preserves the digests + // of the platform manifests it references, so the same platform digest + // addresses the manifest in both repositories. + By("verifying each platform manifest carries its own SBOM in both repositories") + for _, platform := range multiplatformSbomPlatforms { + record, hasRecord := byPlatform[platform] + Expect(hasRecord).To(BeTrue(), "no build report record for platform %s", platform) - platformDigest := record.DockerImageDigest - Expect(platformDigest).NotTo(BeEmpty()) - Expect(platformDigest).NotTo(Equal(indexDigest)) + platformDigest := record.DockerImageDigest + Expect(platformDigest).NotTo(BeEmpty()) + Expect(platformDigest).NotTo(Equal(indexDigest)) - stagesDesc, _ := fetchSingleSbomArtifact(ctx, stagesRepo, platformDigest) - Expect(stagesDesc.Annotations[image.WerfPlatformAnnotation]).To(Equal(platform)) - - finalDesc, _ := fetchSingleSbomArtifact(ctx, finalRepo, platformDigest) - Expect(finalDesc.Annotations[image.WerfPlatformAnnotation]).To(Equal(platform)) + for _, repo := range []string{stagesRepo, finalRepo} { + desc, payload := fetchSingleSbomArtifact(ctx, repo, platformDigest) + Expect(desc.ArtifactType).To(Equal(attestation.DSSEMediaType)) + Expect(desc.Annotations[image.WerfPlatformAnnotation]).To(Equal(platform), + "artifact attached to %s in %s must belong to platform %s", platformDigest, repo, platform) + Expect(desc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + // The artifact describes the platform manifest it is attached to: a copy + // that moved a platform SBOM onto another digest, or the wrong platform's + // SBOM onto this one, differs here and nowhere else. + Expect(mustExtractInTotoSubjectDigest(payload)).To(Equal(platformDigest), + "in-toto subject of the SBOM in %s must name the platform manifest it describes", repo) } + } - By("verifying no SBOM artifact is attached to the index digest in either repository") - expectNoSbomArtifact(ctx, stagesRepo, indexDigest) - expectNoSbomArtifact(ctx, finalRepo, indexDigest) + By("verifying no SBOM artifact is attached to the index digest in either repository") + expectNoSbomArtifact(ctx, stagesRepo, indexDigest) + expectNoSbomArtifact(ctx, finalRepo, indexDigest) - By("reading a platform SBOM from the final repo through the digest reported to the user") - for _, platform := range multiplatformSbomPlatforms { - sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ - CommonOptions: werf.CommonOptions{ - ExtraArgs: []string{ - "--repo", finalRepo, - "--digest", indexDigest, - "--platform", platform, - }, + By("reading a platform SBOM from the final repo through the digest reported to the user") + for _, platform := range multiplatformSbomPlatforms { + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{ + "--repo", finalRepo, + "--digest", indexDigest, + "--platform", platform, }, - }) - sbomtest.MustParseSBOMOutput(sbomOut) - } - }, - Entry("with final repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), - Entry("with final repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), - ) + }, + }) + sbomtest.MustParseSBOMOutput(sbomOut) + } + }) }) From 2b5b0caa3dd4225e8960f2e8d4fa810554620c74 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 02:20:34 +0300 Subject: [PATCH 07/11] test(build, oci, bundles): pin the artifact propagation contract in unit tests The propagation orchestration, the index traversal and the bundle copy were reachable only through e2e suites that need a live registry, so a regression in any of them passed the fast loop. All three now run against an in-process registry: propagateArtifacts over a single-platform image and over an index with per-platform artifacts, CopyAllAttachedArtifacts over an index and over a source that does not hold the manifest, and the bundle copy over an image with and without artifacts. Each fails when its branch is removed. Stop the bundle stub from reporting every image as absent, which made the artifact-carrying branch dead code in the suite, and cover ListUnregenerableArtifacts, which decides whether the secondary-to-primary copy has anything to warn about. Signed-off-by: Radmir Khurum --- pkg/build/build_phase_propagate_test.go | 237 ++++++++++++++++++++++ pkg/deploy/bundles/copy_artifacts_test.go | 145 +++++++++++++ pkg/deploy/bundles/copy_test.go | 16 +- pkg/oci/artifact/copy_test.go | 98 +++++++++ pkg/oci/artifact/unregenerable_test.go | 73 +++++++ 5 files changed, 562 insertions(+), 7 deletions(-) create mode 100644 pkg/build/build_phase_propagate_test.go create mode 100644 pkg/deploy/bundles/copy_artifacts_test.go create mode 100644 pkg/oci/artifact/unregenerable_test.go diff --git a/pkg/build/build_phase_propagate_test.go b/pkg/build/build_phase_propagate_test.go new file mode 100644 index 0000000000..320afdbfe4 --- /dev/null +++ b/pkg/build/build_phase_propagate_test.go @@ -0,0 +1,237 @@ +package build + +import ( + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/build/image" + "github.com/werf/werf/v3/pkg/docker_registry" + imagePkg "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/oci/artifact" + "github.com/werf/werf/v3/pkg/storage" + "github.com/werf/werf/v3/pkg/storage/manager" + "github.com/werf/werf/v3/test/mock" +) + +var _ = Describe("BuildPhase propagateArtifacts", func() { + var ( + server *httptest.Server + stagesRepo string + finalRepo string + cacheRepo string + remoteOpts []remote.Option + ) + + pushImage := func(ctx SpecContext, repo, tag string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + ref, err := name.NewTag(repo + ":" + tag) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + return dgst.String() + } + + pushIndex := func(ctx SpecContext, repo, tag string, platforms []string) (string, []string) { + idx := v1.ImageIndex(empty.Index) + var children []string + + for _, platform := range platforms { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + parsed, err := v1.ParsePlatform(platform) + Expect(err).To(Succeed()) + + idx = mutate.AppendManifests(idx, mutate.IndexAddendum{Add: img, Descriptor: v1.Descriptor{Platform: parsed}}) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + children = append(children, dgst.String()) + } + + ref, err := name.NewTag(repo + ":" + tag) + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(ref, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := idx.Digest() + Expect(err).To(Succeed()) + return dgst.String(), children + } + + copyManifestByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + desc, err := remote.Get(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(toRef, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + return + } + + img, err := desc.Image() + Expect(err).To(Succeed()) + Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + attach := func(ctx SpecContext, repo, digest, payload string) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + Expect(store.Attach(ctx, digest, attestation.DSSEMediaType, []byte(payload), "checksum-"+payload, "", "")).To(Succeed()) + } + + expectAttached := func(ctx SpecContext, repo, digest, payload string) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + content, err := store.GetAttachedContent(ctx, digest, attestation.DSSEMediaType, nil) + Expect(err).To(Succeed(), "no artifact attached to %s in %s", digest, repo) + Expect(content).To(MatchJSON(payload)) + } + + expectNothingAttached := func(ctx SpecContext, repo, digest string) { + idx, err := artifact.PullFallbackIndex(ctx, repo, digest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + Expect(im.Manifests).To(BeEmpty(), "unexpected artifact attached to %s in %s", digest, repo) + } + + newPhase := func(tree *image.ImagesTree, cacheStorages ...storage.StagesStorage) *BuildPhase { + storageManager := &manager.StorageManager{ + StagesStorage: storage.NewRepoStagesStorage(&storage.NewRepoStagesStorageOptions{RepoAddress: stagesRepo}), + CacheStagesStorageList: cacheStorages, + } + return &BuildPhase{ + BasePhase: BasePhase{Conveyor: &Conveyor{imagesTree: tree, StorageManager: storageManager}}, + sbomStep: &sbomStep{}, + } + } + + newImage := func(ctx SpecContext, platform, repo, digest string, final *imagePkg.StageDesc) *image.Image { + img, err := image.NewImage(ctx, platform, "app", image.NoBaseImage, image.ImageOptions{}) + Expect(err).To(Succeed()) + + img.SetContentTagDesc(&imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: repo, RepoDigest: repo + "@" + digest}, + }) + if final != nil { + img.SetFinalContentTagDesc(final) + } + return img + } + + cacheStorage := func(address string) storage.StagesStorage { + s := mock.NewMockStagesStorage(gomock.NewController(GinkgoT())) + s.EXPECT().Address().Return(address).AnyTimes() + s.EXPECT().String().Return(address).AnyTimes() + return s + } + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + stagesRepo = host + "/test/stages" + finalRepo = host + "/test/final" + cacheRepo = host + "/test/cache" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + }) + + AfterEach(func() { + server.Close() + }) + + It("carries the artifacts of a single-platform image into the final repo and the cache repo", func(ctx SpecContext) { + digest := pushImage(ctx, stagesRepo, "v1") + attach(ctx, stagesRepo, digest, `{"scope":"app"}`) + copyManifestByDigest(ctx, stagesRepo, finalRepo, digest) + copyManifestByDigest(ctx, stagesRepo, cacheRepo, digest) + + finalDesc := &imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: finalRepo, RepoDigest: finalRepo + "@" + digest}, + } + + tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) + tree.AppendImageForTests(newImage(ctx, "linux/amd64", stagesRepo, digest, finalDesc)) + + Expect(newPhase(tree, cacheStorage(cacheRepo)).propagateArtifacts(ctx)).To(Succeed()) + + expectAttached(ctx, finalRepo, digest, `{"scope":"app"}`) + expectAttached(ctx, cacheRepo, digest, `{"scope":"app"}`) + }) + + It("leaves the artifacts alone when the image was not published to a final repo", func(ctx SpecContext) { + digest := pushImage(ctx, stagesRepo, "v1") + attach(ctx, stagesRepo, digest, `{"scope":"app"}`) + copyManifestByDigest(ctx, stagesRepo, finalRepo, digest) + + tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) + tree.AppendImageForTests(newImage(ctx, "linux/amd64", stagesRepo, digest, nil)) + + Expect(newPhase(tree).propagateArtifacts(ctx)).To(Succeed()) + + expectNothingAttached(ctx, finalRepo, digest) + }) + + It("carries per-platform artifacts onto the platform manifests and image-level artifacts onto the index", func(ctx SpecContext) { + platforms := []string{"linux/amd64", "linux/arm64"} + indexDigest, children := pushIndex(ctx, stagesRepo, "index", platforms) + + attach(ctx, stagesRepo, indexDigest, `{"scope":"image"}`) + attach(ctx, stagesRepo, children[0], `{"scope":"amd64"}`) + attach(ctx, stagesRepo, children[1], `{"scope":"arm64"}`) + + copyManifestByDigest(ctx, stagesRepo, finalRepo, indexDigest) + + tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) + images := make([]*image.Image, 0, len(platforms)) + for i, platform := range platforms { + img := newImage(ctx, platform, stagesRepo, children[i], nil) + images = append(images, img) + tree.AppendImageForTests(img) + } + + multiImg := image.NewMultiplatformImage("app", images, 0, 1) + multiImg.SetStageDesc(&imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: stagesRepo, RepoDigest: stagesRepo + "@" + indexDigest}, + }) + multiImg.SetFinalStageDesc(&imagePkg.StageDesc{ + StageID: imagePkg.NewStageID("digest", 1), + Info: &imagePkg.Info{Repository: finalRepo, RepoDigest: finalRepo + "@" + indexDigest}, + }) + tree.SetMultiplatformImage(multiImg) + + // A cache repo never holds the index digest, so offering it the image-level + // artifact could only ever warn; the platform manifests do travel there. + Expect(newPhase(tree, cacheStorage(cacheRepo)).propagateArtifacts(ctx)).To(Succeed()) + + expectAttached(ctx, finalRepo, indexDigest, `{"scope":"image"}`) + expectAttached(ctx, finalRepo, children[0], `{"scope":"amd64"}`) + expectAttached(ctx, finalRepo, children[1], `{"scope":"arm64"}`) + expectNothingAttached(ctx, cacheRepo, indexDigest) + }) +}) diff --git a/pkg/deploy/bundles/copy_artifacts_test.go b/pkg/deploy/bundles/copy_artifacts_test.go new file mode 100644 index 0000000000..904b9350ee --- /dev/null +++ b/pkg/deploy/bundles/copy_artifacts_test.go @@ -0,0 +1,145 @@ +package bundles + +import ( + "fmt" + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + chartcommon "github.com/werf/nelm/v2/pkg/helm/pkg/chart/common" + chart "github.com/werf/nelm/v2/pkg/helm/pkg/chart/v2" + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/logging" + "github.com/werf/werf/v3/pkg/oci/artifact" + bundles_registry "github.com/werf/werf/v3/pkg/ref" +) + +var _ = Describe("Bundle copy artifacts", func() { + const imageTag = "tag-1" + + var ( + server *httptest.Server + srcRepo string + dstRepo string + remoteOpts []remote.Option + ) + + pushImage := func(ctx SpecContext, repo string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + ref, err := name.NewTag(repo + ":" + imageTag) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + return dgst.String() + } + + copyImageByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + img, err := remote.Image(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + // copyBundle runs a remote-to-remote bundle copy of a single-image bundle. The + // bundle copy itself goes through the stub, which holds archives rather than + // manifests, so the destination manifest has to be placed separately — at the + // same digest, the way a registry copy places it. + copyBundle := func(ctx SpecContext, digest string) { + srcRef := fmt.Sprintf("%s:%s", srcRepo, imageTag) + ch := &chart.Chart{ + Metadata: &chart.Metadata{ + APIVersion: "v2", + Name: "testproject", + Version: "1.2.3", + Type: "application", + }, + Values: map[string]interface{}{ + "werf": map[string]interface{}{ + "image": map[string]interface{}{"image-1": srcRef}, + "repo": srcRepo, + }, + }, + Raw: []*chartcommon.File{ + { + Name: "values.yaml", + Data: []byte(fmt.Sprintf("werf:\n image:\n image-1: %s\n repo: %s\n", srcRef, srcRepo)), + }, + }, + } + + bundlesRegistryClient := NewBundlesRegistryClientStub() + registryClient := NewDockerRegistryStub() + registryClient.ImagesByReference[srcRef] = []byte(`image-1-bytes`) + registryClient.RepoImagesByReference[srcRef] = &image.Info{RepoDigest: srcRepo + "@" + digest} + + fromAddr, err := bundles_registry.ParseAddr(srcRepo + ":1.2.3") + Expect(err).NotTo(HaveOccurred()) + from := NewRemoteBundle(fromAddr.RegistryAddress, bundlesRegistryClient, registryClient) + bundlesRegistryClient.StubCharts[fromAddr.RegistryAddress.FullName()] = ch + + toAddr, err := bundles_registry.ParseAddr(dstRepo + ":4.5.6") + Expect(err).NotTo(HaveOccurred()) + to := NewRemoteBundle(toAddr.RegistryAddress, bundlesRegistryClient, registryClient) + + Expect(from.CopyTo(logging.WithLogger(ctx), to, copyToOptions{})).To(Succeed()) + } + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + srcRepo = host + "/group/testproject" + dstRepo = host + "/group2/testproject2" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + }) + + AfterEach(func() { + server.Close() + }) + + It("should carry the artifacts of the images it copies between registries", func(ctx SpecContext) { + digest := pushImage(ctx, srcRepo) + + srcStore := artifact.NewOCIStore(srcRepo, "image-1", remoteOpts...) + Expect(srcStore.Attach(ctx, digest, attestation.DSSEMediaType, []byte(`{"sbom":true}`), "checksum-v1", "", "")).To(Succeed()) + + copyImageByDigest(ctx, srcRepo, dstRepo, digest) + copyBundle(ctx, digest) + + dstStore := artifact.NewOCIStore(dstRepo, "image-1", remoteOpts...) + content, err := dstStore.GetAttachedContent(ctx, digest, attestation.DSSEMediaType, nil) + Expect(err).To(Succeed()) + Expect(content).To(MatchJSON(`{"sbom":true}`)) + }) + + It("should copy a bundle whose images carry no artifacts", func(ctx SpecContext) { + digest := pushImage(ctx, srcRepo) + + copyImageByDigest(ctx, srcRepo, dstRepo, digest) + copyBundle(ctx, digest) + + idx, err := artifact.PullFallbackIndex(ctx, dstRepo, digest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + Expect(im.Manifests).To(BeEmpty()) + }) +}) diff --git a/pkg/deploy/bundles/copy_test.go b/pkg/deploy/bundles/copy_test.go index 6cc8c4df48..8e2031269f 100644 --- a/pkg/deploy/bundles/copy_test.go +++ b/pkg/deploy/bundles/copy_test.go @@ -645,12 +645,14 @@ func (client *BundlesRegistryClientStub) PushChart(ctx context.Context, ref *bun type DockerRegistryStub struct { docker_registry.Interface - ImagesByReference map[string][]byte + ImagesByReference map[string][]byte + RepoImagesByReference map[string]*image.Info } func NewDockerRegistryStub() *DockerRegistryStub { return &DockerRegistryStub{ - ImagesByReference: make(map[string][]byte), + ImagesByReference: make(map[string][]byte), + RepoImagesByReference: make(map[string]*image.Info), } } @@ -687,11 +689,11 @@ func (registry *DockerRegistryStub) PullImageArchive(ctx context.Context, archiv return nil } -// TryGetRepoImage reports every image as absent: the stub holds archives, not -// registry manifests, so there are no attached artifacts to resolve and the -// artifact-carrying step is skipped. -func (registry *DockerRegistryStub) TryGetRepoImage(_ context.Context, _ string) (*image.Info, error) { - return nil, nil +// TryGetRepoImage resolves only what a spec put into RepoImagesByReference. The +// stub holds archives rather than registry manifests, so by default an image has +// no digest to carry artifacts for and the artifact-carrying step is skipped. +func (registry *DockerRegistryStub) TryGetRepoImage(_ context.Context, reference string) (*image.Info, error) { + return registry.RepoImagesByReference[reference], nil } func (registry *DockerRegistryStub) CopyImage(_ context.Context, sourceReference, destinationReference string, _ docker_registry.CopyImageOptions) error { diff --git a/pkg/oci/artifact/copy_test.go b/pkg/oci/artifact/copy_test.go index 1470a053aa..af8a77e8a2 100644 --- a/pkg/oci/artifact/copy_test.go +++ b/pkg/oci/artifact/copy_test.go @@ -1,6 +1,7 @@ package artifact_test import ( + "fmt" "net/http/httptest" "strings" @@ -8,6 +9,8 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/go-containerregistry/pkg/v1/remote" . "github.com/onsi/ginkgo/v2" @@ -149,4 +152,99 @@ var _ = Describe("CopyAttachedArtifacts (integration)", func() { im := pullIndex(ctx, dstRepo, srcDigest) Expect(im.Manifests).To(BeEmpty()) }) + + Describe("CopyAllAttachedArtifacts", func() { + // The index carries a platform per manifest the way a werf multi-platform + // image does: entries without one are not platform manifests and are not + // traversed. + pushMultiplatformIndex := func(ctx SpecContext, repo, tag string) (string, []string) { + idx := v1.ImageIndex(empty.Index) + var children []string + + for _, platform := range []string{"linux/amd64", "linux/arm64"} { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + parsed, err := v1.ParsePlatform(platform) + Expect(err).To(Succeed()) + + idx = mutate.AppendManifests(idx, mutate.IndexAddendum{ + Add: img, + Descriptor: v1.Descriptor{Platform: parsed}, + }) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + children = append(children, dgst.String()) + } + + ref, err := name.NewTag(repo + ":" + tag) + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(ref, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := idx.Digest() + Expect(err).To(Succeed()) + + return dgst.String(), children + } + + copyIndexByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + idx, err := remote.Index(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + Expect(remote.WriteIndex(toRef, idx, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + It("should carry the artifacts of every manifest an index references", func(ctx SpecContext) { + indexDigest, children := pushMultiplatformIndex(ctx, srcRepo, "index") + + store := artifact.NewOCIStore(srcRepo, "my-app", remoteOpts...) + Expect(store.Attach(ctx, indexDigest, artifactType, []byte(`{"scope":"image"}`), "checksum-index", "", "")).To(Succeed()) + for i, child := range children { + payload := fmt.Sprintf(`{"scope":"platform-%d"}`, i) + Expect(store.Attach(ctx, child, artifactType, []byte(payload), fmt.Sprintf("checksum-%d", i), "", "")).To(Succeed()) + } + + copyIndexByDigest(ctx, srcRepo, dstRepo, indexDigest) + + Expect(artifact.CopyAllAttachedArtifacts(ctx, srcRepo, indexDigest, dstRepo, indexDigest, remoteOpts...)).To(Succeed()) + + dstStore := artifact.NewOCIStore(dstRepo, "my-app", remoteOpts...) + content, err := dstStore.GetAttachedContent(ctx, indexDigest, artifactType, nil) + Expect(err).To(Succeed()) + Expect(content).To(MatchJSON(`{"scope":"image"}`)) + + for i, child := range children { + childContent, err := dstStore.GetAttachedContent(ctx, child, artifactType, nil) + Expect(err).To(Succeed()) + Expect(childContent).To(MatchJSON(fmt.Sprintf(`{"scope":"platform-%d"}`, i))) + } + }) + + It("should not move an artifact of a referenced manifest onto the index digest", func(ctx SpecContext) { + indexDigest, children := pushMultiplatformIndex(ctx, srcRepo, "index") + + store := artifact.NewOCIStore(srcRepo, "my-app", remoteOpts...) + Expect(store.Attach(ctx, children[0], artifactType, []byte(`{"scope":"platform-0"}`), "checksum-0", "", "")).To(Succeed()) + + copyIndexByDigest(ctx, srcRepo, dstRepo, indexDigest) + + Expect(artifact.CopyAllAttachedArtifacts(ctx, srcRepo, indexDigest, dstRepo, indexDigest, remoteOpts...)).To(Succeed()) + + Expect(pullIndex(ctx, dstRepo, indexDigest).Manifests).To(BeEmpty()) + Expect(pullIndex(ctx, dstRepo, children[0]).Manifests).To(HaveLen(1)) + }) + + It("should be a no-op when the source does not hold the manifest", func(ctx SpecContext) { + absentDigest := pushRandomImage(ctx, dstRepo, "only-in-dst") + + Expect(artifact.CopyAllAttachedArtifacts(ctx, srcRepo, absentDigest, dstRepo, absentDigest, remoteOpts...)).To(Succeed()) + + Expect(pullIndex(ctx, dstRepo, absentDigest).Manifests).To(BeEmpty()) + }) + }) }) diff --git a/pkg/oci/artifact/unregenerable_test.go b/pkg/oci/artifact/unregenerable_test.go new file mode 100644 index 0000000000..fab689e72a --- /dev/null +++ b/pkg/oci/artifact/unregenerable_test.go @@ -0,0 +1,73 @@ +package artifact_test + +import ( + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/oci/artifact" +) + +var _ = Describe("ListUnregenerableArtifacts (integration)", func() { + var ( + server *httptest.Server + repo string + parentDigest string + remoteOpts []remote.Option + ) + + BeforeEach(func(ctx SpecContext) { + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + repo = host + "/test/app" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + + parent, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + parentRef, err := name.NewTag(repo + ":v1") + Expect(err).To(Succeed()) + Expect(remote.Write(parentRef, parent, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := parent.Digest() + Expect(err).To(Succeed()) + parentDigest = dgst.String() + }) + + AfterEach(func() { + server.Close() + }) + + It("should report nothing for a digest without artifacts", func(ctx SpecContext) { + Expect(artifact.ListUnregenerableArtifacts(ctx, repo, parentDigest, remoteOpts...)).To(BeEmpty()) + }) + + It("should not report artifacts a build regenerates", func(ctx SpecContext) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + Expect(store.Attach(ctx, parentDigest, attestation.DSSEMediaType, []byte(`{"sbom":true}`), "checksum-sbom", "linux/amd64", "https://cyclonedx.org/bom")).To(Succeed()) + + Expect(artifact.ListUnregenerableArtifacts(ctx, repo, parentDigest, remoteOpts...)).To(BeEmpty()) + }) + + It("should report an attestation signed outside the build", func(ctx SpecContext) { + store := artifact.NewOCIStore(repo, "app", remoteOpts...) + Expect(store.Attach(ctx, parentDigest, attestation.DSSEMediaType, []byte(`{"sbom":true}`), "checksum-sbom", "", "https://cyclonedx.org/bom")).To(Succeed()) + Expect(store.Attach(ctx, parentDigest, attestation.DSSEMediaType, []byte(`{"custom":true}`), "", "", "https://example.com/predicate/v1")).To(Succeed()) + + idx, err := artifact.PullFallbackIndex(ctx, repo, parentDigest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + Expect(im.Manifests).To(HaveLen(2), "both artifacts must coexist for this spec to discriminate") + + Expect(artifact.ListUnregenerableArtifacts(ctx, repo, parentDigest, remoteOpts...)).To(ConsistOf("https://example.com/predicate/v1")) + }) +}) From e1b643971de08dd95bda5e5eb38d5174d79bb187 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 02:39:51 +0300 Subject: [PATCH 08/11] test(e2e): cover SBOM and VEX across every repository of a build The suite checked --final-repo only on builds that resolve everything in the stages repo, so nothing exercised the path the card is about: stages reused from a secondary repo, a clean stages repo, a final repo and a cache repo that holds none of it. The new spec builds a base-image chain and an import chain that way and reads the merged SBOM and the VEX document back out of both the stages repo and the final repo. The composition assertion compares the merged SBOM against the SBOM of the import source: app is built from a scratch base and declares no packages of its own, so those components can only have come through the merge. The spec fails when the final-repo propagation is skipped. Signed-off-by: Radmir Khurum --- .../_fixtures/final_repo_sweep/Cargo.lock | 16 +++ .../_fixtures/final_repo_sweep/Cargo.toml | 7 ++ .../final_repo_sweep/Dockerfile.builder-base | 4 + .../_fixtures/final_repo_sweep/base/go.mod | 5 + .../_fixtures/final_repo_sweep/base/go.sum | 2 + .../_fixtures/final_repo_sweep/src/main.rs | 3 + .../final_repo_sweep/vex.openvex.json | 15 +++ .../final_repo_sweep/werf-giterminism.yaml | 5 + .../sbom/_fixtures/final_repo_sweep/werf.yaml | 44 +++++++ test/e2e/sbom/final_repo_sweep_test.go | 117 ++++++++++++++++++ test/e2e/sbom/helpers_test.go | 17 +++ 11 files changed, 235 insertions(+) create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml create mode 100644 test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml create mode 100644 test/e2e/sbom/final_repo_sweep_test.go diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock new file mode 100644 index 0000000000..e65f33706f --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "cargo-simple" +version = "0.1.0" +dependencies = [ + "anyhow", +] diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml new file mode 100644 index 0000000000..8e21b0a428 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "cargo-simple" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.86" diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base b/test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base new file mode 100644 index 0000000000..71b5783d93 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/Dockerfile.builder-base @@ -0,0 +1,4 @@ +FROM rust:1.78-slim +COPY --from=golang:1.22-bookworm /usr/local/go /usr/local/go +ENV PATH=/usr/local/go/bin:$PATH +LABEL io.deckhouse.internal.builder=true diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod new file mode 100644 index 0000000000..a51410178d --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.mod @@ -0,0 +1,5 @@ +module example.com/sweep-base + +go 1.21 + +require golang.org/x/text v0.14.0 diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum new file mode 100644 index 0000000000..c9c7c64d2d --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/base/go.sum @@ -0,0 +1,2 @@ +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs b/test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs new file mode 100644 index 0000000000..0b7841726c --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("sbom-final-repo-sweep"); +} diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json b/test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json new file mode 100644 index 0000000000..1d629a538f --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/vex.openvex.json @@ -0,0 +1,15 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://example.com/vex/e2e-test-001", + "author": "e2e-test", + "timestamp": "2024-06-01T00:00:00Z", + "statements": [ + { + "vulnerability": {"name": "CVE-2024-E2E001"}, + "products": [{"@id": "pkg:oci/werf-test-app"}], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "The vulnerable function is never called in this build." + } + ] +} \ No newline at end of file diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml b/test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml new file mode 100644 index 0000000000..9483c3670c --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/werf-giterminism.yaml @@ -0,0 +1,5 @@ +giterminismConfigVersion: 1 +config: + goTemplateRendering: + allowEnvVariables: + - BUILDER_BASE_IMAGE diff --git a/test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml b/test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml new file mode 100644 index 0000000000..14fd1e2fb3 --- /dev/null +++ b/test/e2e/sbom/_fixtures/final_repo_sweep/werf.yaml @@ -0,0 +1,44 @@ +project: werf-test-e2e-sbom-final-repo-sweep +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: carrier +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: / + to: /c + stageDependencies: + packages: + - Cargo.toml + - Cargo.lock +packages: + - type: rust-cargo + workdir: /c +--- +image: base +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: /base + to: /b + stageDependencies: + packages: + - go.mod + - go.sum +packages: + - type: go-mod + workdir: /b +--- +image: app +fromImage: base +git: + - add: / + to: /a +import: + - image: carrier + add: /c/werf.yaml + to: /imported/werf.yaml + before: setup +vex: vex.openvex.json diff --git a/test/e2e/sbom/final_repo_sweep_test.go b/test/e2e/sbom/final_repo_sweep_test.go new file mode 100644 index 0000000000..fdc3089e7c --- /dev/null +++ b/test/e2e/sbom/final_repo_sweep_test.go @@ -0,0 +1,117 @@ +package e2e_build_test + +import ( + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/vex" + "github.com/werf/werf/v3/test/pkg/attestutils" + "github.com/werf/werf/v3/test/pkg/report" + sbomtest "github.com/werf/werf/v3/test/pkg/sbom" + "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/werf" +) + +var _ = Describe("SBOM and VEX across every repository of a build", Label("e2e", "sbom", "final-repo", "sweep"), func() { + It("reuses stages from a secondary repo and still serves SBOM and VEX from the stages and the final repo", func(ctx SpecContext) { + setupSbomBuildEnv() + + repoDirname := "repo_sbom_final_repo_sweep" + SuiteData.InitTestRepo(ctx, repoDirname, "final_repo_sweep") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + for _, envVar := range buildTrustedBuilderBase(ctx, testRepoPath, "sbom-final-repo-sweep-builder") { + key, value, found := strings.Cut(envVar, "=") + Expect(found).To(BeTrue()) + SuiteData.Stubs.SetEnv(key, value) + } + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + reportProject := report.NewProjectWithReport(werfProject) + + secondaryRepo := suite_init.TestRepo(SuiteData.ProjectName + "-secondary") + stagesRepo := suite_init.TestRepo(SuiteData.ProjectName + "-stages") + finalRepo := suite_init.TestRepo(SuiteData.ProjectName + "-final") + cacheRepo := suite_init.TestRepo(SuiteData.ProjectName + "-cache") + + By("seeding a secondary repo with a full build") + SuiteData.Stubs.SetEnv("WERF_REPO", secondaryRepo) + werfProject.Build(ctx, nil) + + By("rebuilding into a clean stages repo, reusing the secondary repo and publishing to a final repo") + SuiteData.Stubs.SetEnv("WERF_REPO", stagesRepo) + SuiteData.Stubs.SetEnv("WERF_SECONDARY_REPO_1", secondaryRepo) + SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) + // A cache repo that holds none of these stages must not break the + // propagation: artifacts of what it does not hold have nowhere to go. + SuiteData.Stubs.SetEnv("WERF_CACHE_REPO", cacheRepo) + + _, buildReport := reportProject.BuildWithReport(ctx, + SuiteData.GetBuildReportPath("sbom_final_repo_sweep.json"), nil) + + for _, imageName := range []string{"base", "carrier", "app"} { + record, found := buildReport.Images[imageName] + Expect(found).To(BeTrue(), "expected image %q in build report", imageName) + Expect(record.StagesSkipped).To(BeTrue(), + "image %q must be reused from the secondary repo by content-based tag, not rebuilt", imageName) + } + + appRecord := buildReport.Images["app"] + Expect(appRecord.DockerRepo).To(Equal(finalRepo)) + appDigest := appRecord.DockerImageDigest + Expect(appDigest).NotTo(BeEmpty()) + + By("the final repo serves this image's own SBOM, and it carries what the base and the import source contributed") + desc, payload := fetchSingleSbomArtifact(ctx, finalRepo, appDigest) + Expect(desc.ArtifactType).To(Equal(attestation.DSSEMediaType)) + Expect(desc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + Expect(mustExtractInTotoSubjectDigest(payload)).To(Equal(appDigest)) + + finalSbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"--repo", finalRepo, "--digest", appDigest}}, + }) + finalBom := sbomtest.MustParseSBOMOutput(finalSbomOut) + assertCarriesImportedComponents(ctx, werfProject, stagesRepo, finalBom) + + By("the stages repo serves the same SBOM on its own, not by proxy of the final repo") + stagesSbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTagOf(ctx, werfProject, "app", nil)}}, + }) + assertCarriesImportedComponents(ctx, werfProject, stagesRepo, sbomtest.MustParseSBOMOutput(stagesSbomOut)) + + By("VEX is available next to the image in both the stages and the final repo") + Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, finalRepo, appDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), + "the final repo must serve the VEX document of the image it publishes") + + stagesDigest := stagesImageDigestOf(ctx, werfProject, "app") + Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, stagesRepo, stagesDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), + "the stages repo must serve the VEX document on its own") + }) +}) + +// assertCarriesImportedComponents checks the SBOM of app contains everything the +// SBOMs of its base image and its import source list. app is a Stapel image with +// no packages directive, so it is never scanned on its own: a component of either +// source can only have reached it through the merge of that image's SBOM. base +// (Go module) and carrier (Cargo) catalog different ecosystems, so each side of +// the merge is discriminated on its own. +func assertCarriesImportedComponents(ctx SpecContext, werfProject *werf.Project, stagesRepo string, appBom *cdx.BOM) { + for _, source := range []string{"base", "carrier"} { + sourceSbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"--repo", stagesRepo, "--tag", stageTagOf(ctx, werfProject, source, nil)}}, + }) + sourceBom := sbomtest.MustParseSBOMOutput(sourceSbomOut) + + Expect(sourceBom.Components).NotTo(BeNil()) + Expect(*sourceBom.Components).NotTo(BeEmpty(), "image %q must contribute components for this assertion to discriminate", source) + + for _, component := range *sourceBom.Components { + Expect(sbomtest.FindComponentByPURL(appBom, component.PackageURL)).NotTo(BeNil(), + "component %s of image %q is missing from the merged SBOM", component.PackageURL, source) + } + } +} diff --git a/test/e2e/sbom/helpers_test.go b/test/e2e/sbom/helpers_test.go index 21968bd186..33a9155370 100644 --- a/test/e2e/sbom/helpers_test.go +++ b/test/e2e/sbom/helpers_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "io" + "os" "slices" "github.com/google/go-containerregistry/pkg/authn" @@ -17,6 +18,7 @@ import ( "github.com/werf/werf/v3/pkg/attestation" "github.com/werf/werf/v3/pkg/oci/artifact" sbomImage "github.com/werf/werf/v3/pkg/sbom/image" + "github.com/werf/werf/v3/test/pkg/werf" ) var multiplatformSbomPlatforms = []string{"linux/amd64", "linux/arm64"} @@ -95,6 +97,21 @@ func expectNoSbomArtifact(ctx SpecContext, repo, parentDigest string) { Expect(dsseDescs).To(BeEmpty(), "no SBOM artifact must be attached to the index digest %s", parentDigest) } +// stagesImageDigestOf resolves the digest of the image's last stage in the stages +// repo, which is what artifacts in the stages and cache repos are attached to. +func stagesImageDigestOf(ctx SpecContext, werfProject *werf.Project, imageName string) string { + stagesRepo := os.Getenv("WERF_REPO") + Expect(stagesRepo).NotTo(BeEmpty()) + + tagRef, err := name.NewTag(stagesRepo+":"+stageTagOf(ctx, werfProject, imageName, nil), name.Insecure) + Expect(err).NotTo(HaveOccurred()) + + desc, err := remote.Get(tagRef, insecureRemoteOptions(ctx)...) + Expect(err).NotTo(HaveOccurred()) + + return desc.Digest.String() +} + func mustExtractInTotoSubjectDigest(dsseEnvelope []byte) string { var envelope struct { Payload []byte `json:"payload"` From fd6d0ed4f4f62f3420aa1bf9d487f181b07f3678 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 09:27:21 +0300 Subject: [PATCH 09/11] test(e2e): give the cleanup spec the git remote werf cleanup requires Signed-off-by: Radmir Khurum --- test/e2e/sbom/final_repo_cleanup_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/e2e/sbom/final_repo_cleanup_test.go b/test/e2e/sbom/final_repo_cleanup_test.go index 1c7677c40f..340b2651ae 100644 --- a/test/e2e/sbom/final_repo_cleanup_test.go +++ b/test/e2e/sbom/final_repo_cleanup_test.go @@ -12,6 +12,7 @@ import ( "github.com/werf/werf/v3/test/pkg/report" sbomtest "github.com/werf/werf/v3/test/pkg/sbom" "github.com/werf/werf/v3/test/pkg/suite_init" + "github.com/werf/werf/v3/test/pkg/utils" "github.com/werf/werf/v3/test/pkg/werf" ) @@ -66,6 +67,12 @@ var _ = Describe("SBOM retention across cleanup", Label("e2e", "sbom", "final-re keepListPath := filepath.Join(testRepoPath, ".werf-keep-list") Expect(os.WriteFile(keepListPath, []byte(stageTag+"\n"), 0o600)).To(Succeed()) + // werf cleanup requires a git remote origin; it lives outside the project + // directory so that it does not turn into an untracked file of the build. + bareRemotePath := filepath.Join(SuiteData.TmpDir, "sbom_final_repo_cleanup_remote.git") + utils.RunSucceedCommand(ctx, testRepoPath, "git", "init", "--bare", bareRemotePath) + utils.RunSucceedCommand(ctx, testRepoPath, "git", "remote", "add", "origin", bareRemotePath) + cleanupArgs := []string{"cleanup", "--without-kube", "--keep-stages-built-within-last-n-hours=0"} By("running cleanup twice while the stage is on the keep list") From 63451289074ba4f74a68155f1af1ab1ad0e11deb Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 12:23:46 +0300 Subject: [PATCH 10/11] test(build, oci, storage): pin the repair, digest-compare and cache paths of propagation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four branches of the artifact propagation contract had no test that could fail: the repair pass of RepoStagesStorage.CopyFromStorage on a destination that already holds the stage, the digest-compare rule of the secondary-to-primary copy, the non-404 side of the source-manifest tolerance in CopyAllAttachedArtifacts, and the per-platform copy into a cache repo, whose only assertion held whether or not the cache was offered anything. Each gets a test that fails under the mutation it names, and the sweep e2e now reads the SBOM and the VEX document back from the cache repo as well: the cache receives every built stage under the stages repo digest, so the artifacts must be there too. The spec set WERF_CACHE_REPO, which werf does not read — the cache repo comes from WERF_CACHE_REPO_ — so no build had used it. Signed-off-by: Radmir Khurum --- pkg/build/build_phase_propagate_test.go | 6 +- pkg/oci/artifact/copy_test.go | 35 ++++ .../storage_manager_copy_suitable_test.go | 172 ++++++++++++++++++ pkg/storage/meta_repo_marker_test.go | 6 +- pkg/storage/repo_stages_storage_copy_test.go | 133 ++++++++++++++ pkg/storage/stage_lookup_test.go | 13 +- test/e2e/sbom/final_repo_sweep_test.go | 11 +- 7 files changed, 366 insertions(+), 10 deletions(-) create mode 100644 pkg/storage/manager/storage_manager_copy_suitable_test.go create mode 100644 pkg/storage/repo_stages_storage_copy_test.go diff --git a/pkg/build/build_phase_propagate_test.go b/pkg/build/build_phase_propagate_test.go index 320afdbfe4..9013084073 100644 --- a/pkg/build/build_phase_propagate_test.go +++ b/pkg/build/build_phase_propagate_test.go @@ -205,6 +205,8 @@ var _ = Describe("BuildPhase propagateArtifacts", func() { attach(ctx, stagesRepo, children[1], `{"scope":"arm64"}`) copyManifestByDigest(ctx, stagesRepo, finalRepo, indexDigest) + copyManifestByDigest(ctx, stagesRepo, cacheRepo, children[0]) + copyManifestByDigest(ctx, stagesRepo, cacheRepo, children[1]) tree := image.NewImagesTree(nil, image.ImagesTreeOptions{}) images := make([]*image.Image, 0, len(platforms)) @@ -225,13 +227,13 @@ var _ = Describe("BuildPhase propagateArtifacts", func() { }) tree.SetMultiplatformImage(multiImg) - // A cache repo never holds the index digest, so offering it the image-level - // artifact could only ever warn; the platform manifests do travel there. Expect(newPhase(tree, cacheStorage(cacheRepo)).propagateArtifacts(ctx)).To(Succeed()) expectAttached(ctx, finalRepo, indexDigest, `{"scope":"image"}`) expectAttached(ctx, finalRepo, children[0], `{"scope":"amd64"}`) expectAttached(ctx, finalRepo, children[1], `{"scope":"arm64"}`) + expectAttached(ctx, cacheRepo, children[0], `{"scope":"amd64"}`) + expectAttached(ctx, cacheRepo, children[1], `{"scope":"arm64"}`) expectNothingAttached(ctx, cacheRepo, indexDigest) }) }) diff --git a/pkg/oci/artifact/copy_test.go b/pkg/oci/artifact/copy_test.go index af8a77e8a2..85ff0edfac 100644 --- a/pkg/oci/artifact/copy_test.go +++ b/pkg/oci/artifact/copy_test.go @@ -2,6 +2,8 @@ package artifact_test import ( "fmt" + "io" + "net/http" "net/http/httptest" "strings" @@ -246,5 +248,38 @@ var _ = Describe("CopyAttachedArtifacts (integration)", func() { Expect(pullIndex(ctx, dstRepo, absentDigest).Manifests).To(BeEmpty()) }) + + It("should fail when the source manifest cannot be read for a reason other than absence", func(ctx SpecContext) { + indexDigest, _ := pushMultiplatformIndex(ctx, srcRepo, "index") + copyIndexByDigest(ctx, srcRepo, dstRepo, indexDigest) + + upstream := server.Listener.Addr().String() + failing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/manifests/"+indexDigest) { + http.Error(w, "registry unavailable", http.StatusInternalServerError) + return + } + r.URL.Scheme = "http" + r.URL.Host = upstream + r.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + for k, vs := range resp.Header { + w.Header()[k] = vs + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + defer failing.Close() + + failingSrcRepo := strings.TrimPrefix(failing.URL, "http://") + "/test/src" + + err := artifact.CopyAllAttachedArtifacts(ctx, failingSrcRepo, indexDigest, dstRepo, indexDigest, remoteOpts...) + Expect(err).To(MatchError(ContainSubstring("list index manifests"))) + }) }) }) diff --git a/pkg/storage/manager/storage_manager_copy_suitable_test.go b/pkg/storage/manager/storage_manager_copy_suitable_test.go new file mode 100644 index 0000000000..abc0f229fe --- /dev/null +++ b/pkg/storage/manager/storage_manager_copy_suitable_test.go @@ -0,0 +1,172 @@ +package manager + +import ( + "context" + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/container_backend" + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/logging" + "github.com/werf/werf/v3/pkg/oci/artifact" + "github.com/werf/werf/v3/pkg/storage" + "github.com/werf/werf/v3/pkg/werf" + "github.com/werf/werf/v3/test/mock" +) + +// copySuitableFakeStorage stands in for a registry-backed stages storage on the +// backend-mediated copy: it answers with the descriptor the stage has in its +// repository and accepts the fetch and the store without touching a backend. +type copySuitableFakeStorage struct { + storage.PrimaryStagesStorage + + address string + desc *image.StageDesc +} + +func (f *copySuitableFakeStorage) Address() string { return f.address } +func (f *copySuitableFakeStorage) String() string { return f.address } + +func (f *copySuitableFakeStorage) ConstructStageImageName(_, digest string, creationTs int64) string { + return f.address + ":" + image.NewStageID(digest, creationTs).String() +} + +func (f *copySuitableFakeStorage) FetchImage(_ context.Context, _ container_backend.LegacyImageInterface) error { + return nil +} + +func (f *copySuitableFakeStorage) StoreImage(_ context.Context, _ container_backend.LegacyImageInterface) error { + return nil +} + +func (f *copySuitableFakeStorage) GetStageDesc(_ context.Context, _ string, _ image.StageID) (*image.StageDesc, error) { + return f.desc, nil +} + +var _ = Describe("StorageManager.CopySuitableStageDescByDigest", func() { + var ( + server *httptest.Server + srcRepo string + dstRepo string + remoteOpts []remote.Option + stageID *image.StageID + ) + + pushRandomImage := func(ctx SpecContext, repo string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + + ref, err := name.NewTag(repo + ":v1") + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + + dgst, err := img.Digest() + Expect(err).To(Succeed()) + return dgst.String() + } + + copyImageByDigest := func(ctx SpecContext, fromRepo, toRepo, digest string) { + fromRef, err := name.NewDigest(fromRepo + "@" + digest) + Expect(err).To(Succeed()) + img, err := remote.Image(fromRef, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...) + Expect(err).To(Succeed()) + + toRef, err := name.NewDigest(toRepo + "@" + digest) + Expect(err).To(Succeed()) + Expect(remote.Write(toRef, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + stageDescIn := func(repo, digest string) *image.StageDesc { + return &image.StageDesc{ + StageID: stageID, + Info: &image.Info{Name: repo + ":" + stageID.String(), Repository: repo, RepoDigest: repo + "@" + digest}, + } + } + + newManager := func(dst *copySuitableFakeStorage) (*StorageManager, container_backend.ContainerBackend) { + backend := mock.NewMockContainerBackend(gomock.NewController(GinkgoT())) + backend.EXPECT().RenameImage(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + return &StorageManager{ProjectName: "project", StagesStorage: dst}, backend + } + + attachedPayloads := func(ctx SpecContext, repo, digest string) []string { + idx, err := artifact.PullFallbackIndex(ctx, repo, digest, remoteOpts...) + Expect(err).To(Succeed()) + im, err := idx.IndexManifest() + Expect(err).To(Succeed()) + + var payloads []string + for _, desc := range im.Manifests { + payloads = append(payloads, desc.Annotations[artifact.PredicateTypeAnnotation]+"|"+desc.Annotations[image.WerfChecksumAnnotation]) + } + return payloads + } + + BeforeEach(func(ctx SpecContext) { + Expect(werf.Init(GinkgoT().TempDir(), GinkgoT().TempDir())).To(Succeed()) + Expect(image.Init()).To(Succeed()) + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + srcRepo = host + "/test/secondary" + dstRepo = host + "/test/primary" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + stageID = image.NewStageID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1) + }) + + AfterEach(func() { + server.Close() + }) + + It("carries the attached artifacts when the copy preserved the digest", func(ctx SpecContext) { + digest := pushRandomImage(ctx, srcRepo) + copyImageByDigest(ctx, srcRepo, dstRepo, digest) + + srcStore := artifact.NewOCIStore(srcRepo, "app", remoteOpts...) + Expect(srcStore.Attach(ctx, digest, attestation.DSSEMediaType, []byte(`{"sbom":1}`), "checksum-sbom", "", "")).To(Succeed()) + + src := ©SuitableFakeStorage{address: srcRepo} + dst := ©SuitableFakeStorage{address: dstRepo, desc: stageDescIn(dstRepo, digest)} + m, backend := newManager(dst) + + desc, err := m.CopySuitableStageDescByDigest(logging.WithLogger(ctx), stageDescIn(srcRepo, digest), src, dst, backend, "linux/amd64") + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(digest)) + + Expect(attachedPayloads(ctx, dstRepo, digest)).To(HaveLen(1)) + }) + + It("carries nothing when the copy changed the digest", func(ctx SpecContext) { + srcDigest := pushRandomImage(ctx, srcRepo) + dstDigest := pushRandomImage(ctx, dstRepo) + Expect(dstDigest).NotTo(Equal(srcDigest)) + + srcStore := artifact.NewOCIStore(srcRepo, "app", remoteOpts...) + Expect(srcStore.Attach(ctx, srcDigest, attestation.DSSEMediaType, []byte(`{"sbom":1}`), "checksum-sbom", "", "")).To(Succeed()) + Expect(srcStore.Attach(ctx, srcDigest, attestation.DSSEMediaType, []byte(`{"signed":1}`), "", "", "https://example.com/user-predicate")).To(Succeed()) + + src := ©SuitableFakeStorage{address: srcRepo} + dst := ©SuitableFakeStorage{address: dstRepo, desc: stageDescIn(dstRepo, dstDigest)} + m, backend := newManager(dst) + + desc, err := m.CopySuitableStageDescByDigest(logging.WithLogger(ctx), stageDescIn(srcRepo, srcDigest), src, dst, backend, "linux/amd64") + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(dstDigest)) + + Expect(attachedPayloads(ctx, dstRepo, dstDigest)).To(BeEmpty(), "an artifact about %s must not be attached to %s", srcDigest, dstDigest) + Expect(attachedPayloads(ctx, dstRepo, srcDigest)).To(BeEmpty()) + Expect(attachedPayloads(ctx, srcRepo, srcDigest)).To(HaveLen(2), "the source keeps what was left behind") + }) +}) diff --git a/pkg/storage/meta_repo_marker_test.go b/pkg/storage/meta_repo_marker_test.go index 7331168bd3..dde533c291 100644 --- a/pkg/storage/meta_repo_marker_test.go +++ b/pkg/storage/meta_repo_marker_test.go @@ -167,7 +167,11 @@ func (r *markerRegistry) CopyImage(_ context.Context, sourceReference, destinati for k, v := range src.Labels { labels[k] = v } - r.images[destinationReference] = &image.Info{Name: destinationReference, Tag: refTag(destinationReference), Labels: labels} + dst := &image.Info{Name: destinationReference, Tag: refTag(destinationReference), Labels: labels} + if src.RepoDigest != "" { + dst.RepoDigest = refRepo(destinationReference) + "@" + strings.SplitN(src.RepoDigest, "@", 2)[1] + } + r.images[destinationReference] = dst return nil } diff --git a/pkg/storage/repo_stages_storage_copy_test.go b/pkg/storage/repo_stages_storage_copy_test.go new file mode 100644 index 0000000000..63386134ca --- /dev/null +++ b/pkg/storage/repo_stages_storage_copy_test.go @@ -0,0 +1,133 @@ +package storage + +import ( + "context" + "net/http/httptest" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v3/pkg/attestation" + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/image" + "github.com/werf/werf/v3/pkg/oci/artifact" +) + +type copyFromStorageRegistryStub struct { + docker_registry.Interface + + repoDigestByReference map[string]string + copied []string +} + +func (r *copyFromStorageRegistryStub) GetRepoImage(_ context.Context, reference string) (*image.Info, error) { + repoDigest, found := r.repoDigestByReference[reference] + if !found { + return nil, &manifestUnknownError{} + } + return &image.Info{Name: reference, RepoDigest: repoDigest}, nil +} + +func (r *copyFromStorageRegistryStub) IsTagExist(_ context.Context, _ string, _ ...docker_registry.Option) (bool, error) { + return false, nil +} + +func (r *copyFromStorageRegistryStub) CopyImage(_ context.Context, sourceReference, destinationReference string, _ docker_registry.CopyImageOptions) error { + r.copied = append(r.copied, sourceReference+" -> "+destinationReference) + return nil +} + +type manifestUnknownError struct{} + +func (e *manifestUnknownError) Error() string { return "MANIFEST_UNKNOWN: manifest unknown" } + +var _ = Describe("RepoStagesStorage.CopyFromStorage", func() { + const projectName = "project" + + var ( + server *httptest.Server + srcRepo string + dstRepo string + remoteOpts []remote.Option + stageID image.StageID + ) + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + host := strings.TrimPrefix(server.URL, "http://") + srcRepo = host + "/test/src" + dstRepo = host + "/test/dst" + remoteOpts = []remote.Option{remote.WithAuth(authn.Anonymous)} + stageID = *image.NewStageID("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1) + }) + + AfterEach(func() { + server.Close() + }) + + pushStage := func(ctx SpecContext, repos ...string) string { + img, err := random.Image(256, 1) + Expect(err).To(Succeed()) + dgst, err := img.Digest() + Expect(err).To(Succeed()) + + for _, repo := range repos { + ref, err := name.NewDigest(repo + "@" + dgst.String()) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, append([]remote.Option{remote.WithContext(ctx)}, remoteOpts...)...)).To(Succeed()) + } + + return dgst.String() + } + + newStorage := func(repo string, stub *copyFromStorageRegistryStub) *RepoStagesStorage { + return NewRepoStagesStorage(&NewRepoStagesStorageOptions{ + RepoAddress: repo, + DockerRegistry: stub, + SkipMetaCheck: true, + }) + } + + It("attaches the artifacts of the source to a destination that already holds the stage without them", func(ctx SpecContext) { + digest := pushStage(ctx, srcRepo, dstRepo) + srcStore := artifact.NewOCIStore(srcRepo, "app", remoteOpts...) + Expect(srcStore.Attach(ctx, digest, attestation.DSSEMediaType, []byte(`{"v":1}`), "checksum-v1", "", "")).To(Succeed()) + + src := newStorage(srcRepo, ©FromStorageRegistryStub{}) + stub := ©FromStorageRegistryStub{repoDigestByReference: map[string]string{ + dstRepo + ":" + stageID.String(): dstRepo + "@" + digest, + }} + dst := newStorage(dstRepo, stub) + + desc, err := dst.CopyFromStorage(ctx, src, projectName, stageID, CopyFromStorageOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(digest)) + Expect(stub.copied).To(BeEmpty(), "the manifest was in place, nothing to copy") + + dstStore := artifact.NewOCIStore(dstRepo, "app", remoteOpts...) + content, err := dstStore.GetAttachedContent(ctx, digest, attestation.DSSEMediaType, nil) + Expect(err).To(Succeed()) + Expect(content).To(MatchJSON(`{"v":1}`)) + }) + + It("does not fail on a destination that already holds a stage the source does not", func(ctx SpecContext) { + digest := pushStage(ctx, dstRepo) + + src := newStorage(srcRepo, ©FromStorageRegistryStub{}) + dst := newStorage(dstRepo, ©FromStorageRegistryStub{repoDigestByReference: map[string]string{ + dstRepo + ":" + stageID.String(): dstRepo + "@" + digest, + }}) + + desc, err := dst.CopyFromStorage(ctx, src, projectName, stageID, CopyFromStorageOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(desc.Info.GetDigest()).To(Equal(digest)) + }) +}) diff --git a/pkg/storage/stage_lookup_test.go b/pkg/storage/stage_lookup_test.go index 15b81b76d0..5f4b5f4210 100644 --- a/pkg/storage/stage_lookup_test.go +++ b/pkg/storage/stage_lookup_test.go @@ -13,6 +13,11 @@ import ( "github.com/werf/werf/v3/pkg/oci/artifact" ) +// stageLookupDigest is a syntactically valid digest for stages the stub registry +// fabricates: the artifact copy addresses the stage by digest and rejects an +// empty one before reaching the registry. +const stageLookupDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" + var _ = ginkgo.Describe("stage lookup", func() { ginkgo.DescribeTable("returns an unavailable error instead of a nil descriptor", func(ctx ginkgo.SpecContext, present, rejected bool, expected error) { @@ -57,8 +62,8 @@ var _ = ginkgo.Describe("stage lookup", func() { stageID := image.NewStageID("digest", 1) sourceRef := source.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) destinationRef := destination.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) - registry.put(sourceRef, nil) - registry.put(destinationRef, nil) + putWithDigest(registry.markerRegistry, sourceRef, source.RepoAddress+"@"+stageLookupDigest) + putWithDigest(registry.markerRegistry, destinationRef, destination.RepoAddress+"@"+stageLookupDigest) registry.brokenImage = registry.images[destinationRef] desc, err := destination.CopyFromStorage(ctx, source, "project", *stageID, CopyFromStorageOptions{}) @@ -78,9 +83,9 @@ var _ = ginkgo.Describe("stage lookup", func() { stageID := image.NewStageID("digest", 1) sourceRef := source.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) destinationRef := destination.ConstructStageImageName("project", stageID.Digest, stageID.CreationTs) - registry.put(sourceRef, nil) + putWithDigest(registry.markerRegistry, sourceRef, source.RepoAddress+"@"+stageLookupDigest) if existing { - registry.put(destinationRef, nil) + putWithDigest(registry.markerRegistry, destinationRef, destination.RepoAddress+"@"+stageLookupDigest) } if rejected { registry.put(makeRepoRejectedStageImageRecord(destination.RepoAddress, stageID.Digest, stageID.CreationTs), nil) diff --git a/test/e2e/sbom/final_repo_sweep_test.go b/test/e2e/sbom/final_repo_sweep_test.go index fdc3089e7c..f42d7e97f1 100644 --- a/test/e2e/sbom/final_repo_sweep_test.go +++ b/test/e2e/sbom/final_repo_sweep_test.go @@ -46,9 +46,7 @@ var _ = Describe("SBOM and VEX across every repository of a build", Label("e2e", SuiteData.Stubs.SetEnv("WERF_REPO", stagesRepo) SuiteData.Stubs.SetEnv("WERF_SECONDARY_REPO_1", secondaryRepo) SuiteData.Stubs.SetEnv("WERF_FINAL_REPO", finalRepo) - // A cache repo that holds none of these stages must not break the - // propagation: artifacts of what it does not hold have nowhere to go. - SuiteData.Stubs.SetEnv("WERF_CACHE_REPO", cacheRepo) + SuiteData.Stubs.SetEnv("WERF_CACHE_REPO_1", cacheRepo) _, buildReport := reportProject.BuildWithReport(ctx, SuiteData.GetBuildReportPath("sbom_final_repo_sweep.json"), nil) @@ -90,6 +88,13 @@ var _ = Describe("SBOM and VEX across every repository of a build", Label("e2e", stagesDigest := stagesImageDigestOf(ctx, werfProject, "app") Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, stagesRepo, stagesDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), "the stages repo must serve the VEX document on its own") + + By("the cache repo holds the stage under the stages repo digest and serves its SBOM and VEX") + cacheDesc, cachePayload := fetchSingleSbomArtifact(ctx, cacheRepo, stagesDigest) + Expect(cacheDesc.Annotations[image.WerfImageNameAnnotation]).To(Equal("app")) + Expect(mustExtractInTotoSubjectDigest(cachePayload)).To(Equal(stagesDigest)) + Expect(attestutils.FindArtifactDescriptorByPredicate(ctx, cacheRepo, stagesDigest, vex.DSSEMediaType, vex.VEXPredicateURI)).NotTo(BeNil(), + "the cache repo must serve the VEX document of the stage it mirrors") }) }) From 5028b1e3b0b2e805bc04809b47d82f68a43c36a7 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Fri, 25 Sep 2026 15:33:30 +0300 Subject: [PATCH 11/11] chore(build, oci, storage): trim propagation comments Rename a variable shadowing the imported types package and drop comments that restated the code. Signed-off-by: Radmir Khurum --- pkg/build/build_phase.go | 2 -- pkg/oci/artifact/fallback.go | 6 +++--- pkg/storage/manager/storage_manager.go | 9 +++------ pkg/storage/repo_stages_storage.go | 10 +++------- 4 files changed, 9 insertions(+), 18 deletions(-) diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 5ae555c576..2c2b883250 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -734,8 +734,6 @@ func (phase *BuildPhase) propagateArtifacts(ctx context.Context) error { continue } - // An image built out of no stages has no descriptor and nothing attached to - // propagate. stageDesc := images[0].GetContentTagDesc() if stageDesc == nil { continue diff --git a/pkg/oci/artifact/fallback.go b/pkg/oci/artifact/fallback.go index e9d6e31d4e..11f0cc3b2c 100644 --- a/pkg/oci/artifact/fallback.go +++ b/pkg/oci/artifact/fallback.go @@ -321,7 +321,7 @@ func ListUnregenerableArtifacts(ctx context.Context, repo, parentDigest string, return nil, fmt.Errorf("read fallback index manifest: %w", err) } - var types []string + var predicateTypes []string for _, desc := range im.Manifests { if desc.ArtifactType == "" || desc.Annotations[image.WerfChecksumAnnotation] != "" { continue @@ -330,10 +330,10 @@ func ListUnregenerableArtifacts(ctx context.Context, repo, parentDigest string, if predicateType == "" { predicateType = desc.ArtifactType } - types = append(types, predicateType) + predicateTypes = append(predicateTypes, predicateType) } - return lo.Uniq(types), nil + return lo.Uniq(predicateTypes), nil } func multipleArtifactEntriesWarning(parentDigest string, matches []v1.Descriptor) string { diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index be6c331206..978fa1738b 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -825,12 +825,9 @@ func (m *StorageManager) CopySuitableStageDescByDigest(ctx context.Context, stag return nil, fmt.Errorf("unable to get stage %s description from %s: %w", stageDesc.StageID.String(), destinationStagesStorage.String(), err) } else { if sourceStagesStorage.Address() != storage.LocalStorageAddress && destinationStagesStorage.Address() != storage.LocalStorageAddress { - // The backend-mediated copy does not guarantee digest preservation. When the - // digest survived, the attached artifacts still describe the copied image and - // are carried byte-identical. When it changed, a statement about the source - // digest is not a statement about the destination digest: werf-generated - // artifacts are regenerated by convergence against the new digest in the same - // run, and only artifacts werf cannot regenerate are worth reporting. + // The backend-mediated copy does not guarantee digest preservation. Artifacts + // describe the source digest; when it changed, werf regenerates its own by + // convergence and only what it cannot regenerate is worth reporting. if destinationStageDesc.Info.GetDigest() == stageDesc.Info.GetDigest() { if err := artifact.CopyAllAttachedArtifacts(ctx, sourceStagesStorage.Address(), stageDesc.Info.GetDigest(), destinationStagesStorage.Address(), destinationStageDesc.Info.GetDigest()); err != nil { return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageDesc.StageID.String(), err) diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 955cbaa8e1..286748f132 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -946,13 +946,9 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } if desc != nil { - // The manifest is already in place, but its artifacts may not be: an earlier - // run could have copied the stage and failed before the artifacts, or the - // artifacts could have appeared in the source afterwards. The copy is - // idempotent, so repeating it here repairs such a destination. Both - // repositories are addressed by the same digest because a stage reaches this - // destination through a registry-level copy, which preserves it; a source that - // does not hold that digest is a no-op rather than a failure. + // The manifest may be in place without its artifacts; the copy is idempotent + // and repairs that. The digest is the same on both sides because a stage + // reaches this destination through a registry-level copy. if err := artifact.CopyAllAttachedArtifacts(ctx, src.Address(), desc.Info.GetDigest(), storage.RepoAddress, desc.Info.GetDigest()); err != nil { return nil, fmt.Errorf("unable to copy artifacts attached to stage %s: %w", stageID, err) }