From ff5cb9db6349e604a5ee093ff1970aa55e349f3f Mon Sep 17 00:00:00 2001 From: Kieran Mann Date: Tue, 6 Oct 2026 20:31:37 -0700 Subject: [PATCH] feat(earthlyoutputs): serve original content of digest-pinned images A local registry export of a pulled image is a rebuilt image: its config, and so its manifest digest, differ from upstream. A client asking for the upstream reference by digest (e.g. compose with `image: alpine@sha256:...`) can therefore not be served locally and has to reach the upstream registry. Add the `export-image-local-registry-source` ref metadata key. When set to the digest-pinned reference an image was pulled from, the exporter also registers the original index, platform manifest and config with the embedded registry. Unmodified layers keep their digests and are already served, so the pinned reference can be pulled from the embedded registry in full, e.g. by a docker daemon using it as a registry mirror. The original content is read from the content store, where BuildKit keeps it leased to the pulled image's cache record, so no network is needed in the common case; anything missing is fetched with the session's credentials. Failure is logged and not fatal: clients fall back to the upstream registry. Advertise support via the earthly.localregistry.source capability, and report unknown manifests and blobs as 404 rather than 500, as most requests to a mirror are for content it does not have. Co-Authored-By: Claude Opus 5.5 --- exporter/earthlyoutputs/export.go | 46 ++++++ .../registry/eodriver/eodriver.go | 16 ++- .../registry/eodriver/eodriver_test.go | 25 ++++ exporter/earthlyoutputs/source.go | 124 +++++++++++++++++ exporter/earthlyoutputs/source_test.go | 131 ++++++++++++++++++ solver/pb/caps.go | 11 ++ 6 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 exporter/earthlyoutputs/registry/eodriver/eodriver_test.go create mode 100644 exporter/earthlyoutputs/source.go create mode 100644 exporter/earthlyoutputs/source_test.go diff --git a/exporter/earthlyoutputs/export.go b/exporter/earthlyoutputs/export.go index 3e9e233ed..291180609 100644 --- a/exporter/earthlyoutputs/export.go +++ b/exporter/earthlyoutputs/export.go @@ -28,12 +28,14 @@ import ( "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/session/pullping" "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/push" + "github.com/moby/buildkit/util/resolver" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -203,6 +205,9 @@ type imgData struct { // localRegExport is set when the image should be exported via local registry. The value // represents the image name that can be used to pull the image from the local registry. localRegExport string + // localRegSource is the digest-pinned image a local registry export was pulled from, + // if any. See keyLocalRegistrySource. + localRegSource reference.Canonical // shouldPush is set when the image should be pushed. shouldPush bool @@ -288,6 +293,13 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source eilr = string(simpleMd["export-image-local-registry"]) hasAnyLocalRegExport = true } + var eilrSrc reference.Canonical + if v := string(simpleMd[keyLocalRegistrySource]); v != "" && eilr != "" { + eilrSrc, err = parseSourceRef(v) + if err != nil { + return nil, nil, err + } + } sp := false if string(simpleMd["export-image-push"]) == "true" { isImage = true @@ -332,6 +344,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } img.localExport = img.localExport || le img.localRegExport = eilr + img.localRegSource = eilrSrc img.shouldPush = img.shouldPush || sp img.insecurePush = img.insecurePush || ip img.opts = opts @@ -550,6 +563,10 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } } + if img.localRegSource != nil { + e.addLocalRegSource(ctx, sessionID, img) + } + if img.shouldPush { err := push.Push( ctx, e.opt.SessionManager, sessionID, img.mp, @@ -641,6 +658,35 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source return resp, nil, nil } +// addLocalRegSource serves the original content of img.localRegSource from the +// embedded registry as part of img. This is best effort: on failure, clients +// pulling the source reference fall back to the upstream registry. +func (e *imageExporterInstance) addLocalRegSource(ctx context.Context, sessionID string, img *imgData) { + cs := e.opt.ImageWriter.ContentStore() + err := func() error { + platform, err := imagePlatform(ctx, cs, *img.mfstDesc) + if err != nil { + return err + } + ref := img.localRegSource.String() + rslvr := resolver.DefaultPool.GetResolver(e.opt.RegistryHosts, ref, "pull", e.opt.SessionManager, session.NewGroup(sessionID)) + descs, err := sourceContent(ctx, cs, rslvr, img.localRegSource, platform) + if err != nil { + return err + } + mmp := eodriver.MultiMultiProviderSingleton + for _, desc := range descs { + if err := mmp.AddImgSub(img.localRegExport, desc.Digest, cs); err != nil { + return err + } + } + return nil + }() + if err != nil { + bklog.G(ctx).WithError(err).Warnf("not serving original content of %s from the local registry", img.localRegSource) + } +} + func (e *imageExporterInstance) Config() *exporter.Config { return exporter.NewConfigWithCompression(e.opts.RefCfg.Compression) } diff --git a/exporter/earthlyoutputs/registry/eodriver/eodriver.go b/exporter/earthlyoutputs/registry/eodriver/eodriver.go index 7928290ca..d3ab8f121 100644 --- a/exporter/earthlyoutputs/registry/eodriver/eodriver.go +++ b/exporter/earthlyoutputs/registry/eodriver/eodriver.go @@ -9,6 +9,7 @@ import ( "time" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" @@ -166,7 +167,7 @@ func (d *driver) get(ctx context.Context, path string, offset int64) (io.ReadClo fullImgName := fmt.Sprintf("%s:%s", imgName, tag) _, baseDigest, err := d.mmp.Get(ctx, fullImgName) if err != nil { - return nil, 0, errors.Wrapf(err, "get %s", path) + return nil, 0, notFoundOr(err, path) } return stringReadCloserOffset(baseDigest.String(), offset) case "revisions": @@ -204,7 +205,7 @@ func (d *driver) get(ctx context.Context, path string, offset int64) (io.ReadClo } ra, err := d.mmp.ReaderAt(ctx, desc) if err != nil { - return nil, 0, errors.Wrapf(err, "blob %s", path) + return nil, 0, notFoundOr(err, path) } return &readerAtReadCloser{ ra: ra, @@ -294,6 +295,17 @@ func (fi fileInfo) IsDir() bool { return false } +// notFoundOr maps a not found error to storagedriver.PathNotFoundError, which +// the registry reports as an unknown manifest or blob (404) rather than an +// internal error (500). This matters when the registry is used as a mirror: +// most requests are for content it does not have. +func notFoundOr(err error, path string) error { + if errdefs.IsNotFound(err) { + return storagedriver.PathNotFoundError{Path: path, DriverName: driverName} + } + return errors.Wrapf(err, "get %s", path) +} + func stringReadCloserOffset(str string, offset int64) (io.ReadCloser, int64, error) { dt := []byte(str) if offset > int64(len(dt)) { diff --git a/exporter/earthlyoutputs/registry/eodriver/eodriver_test.go b/exporter/earthlyoutputs/registry/eodriver/eodriver_test.go new file mode 100644 index 000000000..59f9efb17 --- /dev/null +++ b/exporter/earthlyoutputs/registry/eodriver/eodriver_test.go @@ -0,0 +1,25 @@ +package eodriver + +import ( + "context" + "testing" + + storagedriver "github.com/docker/distribution/registry/storage/driver" + digest "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +// Unknown content must be reported as not found, so that the registry answers +// 404 rather than 500, e.g. when used as a mirror. +func TestGetUnknownIsPathNotFound(t *testing.T) { + d := &driver{mmp: NewMultiMultiProvider()} + sha := digest.FromString("unknown").Encoded() + + for _, path := range []string{ + "/docker/registry/v2/repositories/library/alpine/_manifests/tags/latest/current/link", + "/docker/registry/v2/blobs/sha256/" + sha[:2] + "/" + sha + "/data", + } { + _, _, err := d.get(context.Background(), path, 0) + require.ErrorAs(t, err, &storagedriver.PathNotFoundError{}, path) + } +} diff --git a/exporter/earthlyoutputs/source.go b/exporter/earthlyoutputs/source.go new file mode 100644 index 000000000..89b54b757 --- /dev/null +++ b/exporter/earthlyoutputs/source.go @@ -0,0 +1,124 @@ +package earthlyoutputs + +import ( + "context" + "encoding/json" + "sync" + + "github.com/containerd/containerd/content" + cerrdefs "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes" + "github.com/docker/distribution/reference" + "github.com/moby/buildkit/util/imageutil" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// keyLocalRegistrySource is the per-ref metadata key naming the image a local +// registry export was pulled from, as a digest-pinned reference. When set, the +// original index, manifest and config of that image are served from the +// embedded registry alongside the exported image, so that a client can pull +// the pinned reference itself (e.g. via a registry mirror) and have its digest +// verify. Layers are not added: unmodified layers keep their original digests +// and are already served as part of the exported image. +const keyLocalRegistrySource = "export-image-local-registry-source" + +// parseSourceRef parses the value of keyLocalRegistrySource. +func parseSourceRef(s string) (reference.Canonical, error) { + named, err := reference.ParseNormalizedNamed(s) + if err != nil { + return nil, errors.Wrapf(err, "parse %s %q", keyLocalRegistrySource, s) + } + canonical, ok := named.(reference.Canonical) + if !ok { + return nil, errors.Errorf("%s %q is not pinned by digest", keyLocalRegistrySource, s) + } + return canonical, nil +} + +// sourceContent makes sure the non-layer content (index, the manifest matching +// platform, and its config) of the image ref is in cs, and returns its +// descriptors. Content already in cs is used as is: images pulled by BuildKit +// keep this content leased to their cache records, so the common case needs no +// network access. Anything missing is fetched with resolver. +func sourceContent(ctx context.Context, cs content.Store, resolver remotes.Resolver, ref reference.Canonical, platform ocispecs.Platform) ([]ocispecs.Descriptor, error) { + root, err := rootDescriptor(ctx, cs, resolver, ref) + if err != nil { + return nil, err + } + + fetcher, err := resolver.Fetcher(ctx, ref.String()) + if err != nil { + return nil, errors.Wrapf(err, "fetcher for %s", ref) + } + + matcher := platforms.Only(platform) + children := images.LimitManifests(images.FilterPlatforms(images.ChildrenHandler(cs), matcher), matcher, 1) + + var ( + mu sync.Mutex + descs []ocispecs.Descriptor + ) + collect := images.HandlerFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + if images.IsLayerType(desc.MediaType) { + return nil, images.ErrSkipDesc + } + mu.Lock() + descs = append(descs, desc) + mu.Unlock() + return nil, nil + }) + + // FetchHandler is a no-op for content that is already in cs. + if err := images.Dispatch(ctx, images.Handlers(collect, remotes.FetchHandler(cs, fetcher), children), nil, root); err != nil { + return nil, errors.Wrapf(err, "fetch %s", ref) + } + return descs, nil +} + +// rootDescriptor returns the descriptor ref's digest points to, from cs if +// present there, otherwise by resolving ref. +func rootDescriptor(ctx context.Context, cs content.Store, resolver remotes.Resolver, ref reference.Canonical) (ocispecs.Descriptor, error) { + info, err := cs.Info(ctx, ref.Digest()) + switch { + case err == nil: + desc := ocispecs.Descriptor{Digest: info.Digest, Size: info.Size} + dt, err := content.ReadBlob(ctx, cs, desc) + if err != nil { + return ocispecs.Descriptor{}, errors.Wrapf(err, "read %s", ref) + } + desc.MediaType, err = imageutil.DetectManifestBlobMediaType(dt) + if err != nil { + return ocispecs.Descriptor{}, errors.Wrapf(err, "detect media type of %s", ref) + } + return desc, nil + case cerrdefs.IsNotFound(err): + _, desc, err := resolver.Resolve(ctx, ref.String()) + if err != nil { + return ocispecs.Descriptor{}, errors.Wrapf(err, "resolve %s", ref) + } + return desc, nil + default: + return ocispecs.Descriptor{}, errors.Wrapf(err, "stat %s", ref) + } +} + +// imagePlatform returns the platform of the image described by the manifest +// desc, as recorded in its config. +func imagePlatform(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor) (ocispecs.Platform, error) { + mfst, err := images.Manifest(ctx, provider, desc, platforms.All) + if err != nil { + return ocispecs.Platform{}, errors.Wrap(err, "read manifest") + } + dt, err := content.ReadBlob(ctx, provider, mfst.Config) + if err != nil { + return ocispecs.Platform{}, errors.Wrap(err, "read config") + } + var img ocispecs.Image + if err := json.Unmarshal(dt, &img); err != nil { + return ocispecs.Platform{}, errors.Wrap(err, "unmarshal config") + } + return platforms.Normalize(img.Platform), nil +} diff --git a/exporter/earthlyoutputs/source_test.go b/exporter/earthlyoutputs/source_test.go new file mode 100644 index 000000000..0f8e39b04 --- /dev/null +++ b/exporter/earthlyoutputs/source_test.go @@ -0,0 +1,131 @@ +package earthlyoutputs + +import ( + "bytes" + "context" + "encoding/json" + "io" + "testing" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/remotes" + digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +var errNoNetwork = errors.New("network access not allowed in this test") + +// offlineResolver fails any attempt to reach a registry. +type offlineResolver struct{} + +func (offlineResolver) Resolve(context.Context, string) (string, ocispecs.Descriptor, error) { + return "", ocispecs.Descriptor{}, errNoNetwork +} + +func (offlineResolver) Fetcher(context.Context, string) (remotes.Fetcher, error) { + return remotes.FetcherFunc(func(context.Context, ocispecs.Descriptor) (io.ReadCloser, error) { + return nil, errNoNetwork + }), nil +} + +func (offlineResolver) Pusher(context.Context, string) (remotes.Pusher, error) { + return nil, errNoNetwork +} + +func writeJSON(t *testing.T, cs content.Store, mediaType string, v interface{}) ocispecs.Descriptor { + t.Helper() + dt, err := json.Marshal(v) + require.NoError(t, err) + desc := ocispecs.Descriptor{MediaType: mediaType, Digest: digest.FromBytes(dt), Size: int64(len(dt))} + require.NoError(t, content.WriteBlob(context.Background(), cs, desc.Digest.String(), bytes.NewReader(dt), desc)) + return desc +} + +// writeImage writes a two-platform image index to cs, with only the amd64 +// manifest and config present, as BuildKit leaves it after an amd64 pull. +func writeImage(t *testing.T, cs content.Store) (index, amd64Mfst, amd64Cfg ocispecs.Descriptor) { + t.Helper() + layer := ocispecs.Descriptor{MediaType: ocispecs.MediaTypeImageLayerGzip, Digest: digest.FromString("layer"), Size: 5} + + amd64 := ocispecs.Platform{OS: "linux", Architecture: "amd64"} + amd64Cfg = writeJSON(t, cs, ocispecs.MediaTypeImageConfig, ocispecs.Image{Platform: amd64}) + amd64Mfst = writeJSON(t, cs, ocispecs.MediaTypeImageManifest, ocispecs.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispecs.MediaTypeImageManifest, + Config: amd64Cfg, + Layers: []ocispecs.Descriptor{layer}, + }) + amd64Mfst.Platform = &amd64 + + // Never written to cs: fetching it would need the network. + arm64Mfst := ocispecs.Descriptor{ + MediaType: ocispecs.MediaTypeImageManifest, + Digest: digest.FromString("arm64 manifest"), + Size: 14, + Platform: &ocispecs.Platform{OS: "linux", Architecture: "arm64", Variant: "v8"}, + } + + index = writeJSON(t, cs, ocispecs.MediaTypeImageIndex, ocispecs.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispecs.MediaTypeImageIndex, + Manifests: []ocispecs.Descriptor{arm64Mfst, amd64Mfst}, + }) + return index, amd64Mfst, amd64Cfg +} + +func TestSourceContentFromContentStore(t *testing.T) { + ctx := context.Background() + cs, err := local.NewStore(t.TempDir()) + require.NoError(t, err) + index, mfst, cfg := writeImage(t, cs) + + ref, err := parseSourceRef("alpine@" + index.Digest.String()) + require.NoError(t, err) + + descs, err := sourceContent(ctx, cs, offlineResolver{}, ref, ocispecs.Platform{OS: "linux", Architecture: "amd64"}) + require.NoError(t, err) + + var got []digest.Digest + for _, desc := range descs { + got = append(got, desc.Digest) + } + require.ElementsMatch(t, []digest.Digest{index.Digest, mfst.Digest, cfg.Digest}, got) +} + +func TestSourceContentMissingRootResolves(t *testing.T) { + ctx := context.Background() + cs, err := local.NewStore(t.TempDir()) + require.NoError(t, err) + + ref, err := parseSourceRef("alpine@" + digest.FromString("not in the store").String()) + require.NoError(t, err) + + _, err = sourceContent(ctx, cs, offlineResolver{}, ref, ocispecs.Platform{OS: "linux", Architecture: "amd64"}) + require.ErrorIs(t, err, errNoNetwork) +} + +func TestImagePlatform(t *testing.T) { + ctx := context.Background() + cs, err := local.NewStore(t.TempDir()) + require.NoError(t, err) + _, mfst, _ := writeImage(t, cs) + + got, err := imagePlatform(ctx, cs, mfst) + require.NoError(t, err) + require.Equal(t, ocispecs.Platform{OS: "linux", Architecture: "amd64"}, got) +} + +func TestParseSourceRef(t *testing.T) { + dgst := digest.FromString("x").String() + + ref, err := parseSourceRef("alpine:3.24.2@" + dgst) + require.NoError(t, err) + require.Equal(t, "docker.io/library/alpine:3.24.2@"+dgst, ref.String()) + + _, err = parseSourceRef("alpine:3.24.2") + require.ErrorContains(t, err, "not pinned by digest") +} diff --git a/solver/pb/caps.go b/solver/pb/caps.go index 06e41c735..fa1ffd028 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -92,6 +92,11 @@ const ( CapSourcePolicy apicaps.CapID = "source.policy" CapEarthlyRegistryProxy apicaps.CapID = "earthly.regproxy" // earthly-specific + + // CapEarthlyLocalRegistrySource is the capability of the earthly exporter to + // serve the original content of a digest-pinned image from the embedded + // registry. See exporter/earthlyoutputs. earthly-specific. + CapEarthlyLocalRegistrySource apicaps.CapID = "earthly.localregistry.source" ) func init() { @@ -495,4 +500,10 @@ func init() { Enabled: true, Status: apicaps.CapStatusExperimental, }) + + Caps.Init(apicaps.Cap{ + ID: CapEarthlyLocalRegistrySource, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) }