chore: update buildkit to v0.14.1

.coderabbit.yaml

@@ -0,0 +1,4 @@
+reviews:
+  path_filters:
+    - "!vendor/**"
+    - "!**/vendor/**"

AGENTS.md

@@ -2,6 +2,19 @@
 
 This repository is a fork of [moby/buildkit](https://github.com/moby/buildkit). Its primary use is by [earthbuild/earthbuild](https://github.com/earthbuild/earthbuild).
 
+## Naming
+
+The project is being renamed from Earthly to EarthBuild. When writing new code,
+comments, docs or commit messages:
+
+- Use `earth` for the CLI / build tool (not `earthly`).
+- Use `EarthBuild` for the project name (not `Earthly`).
+- Leave existing `earthly` identifiers alone where they are literal: Earthfile
+  target names (`+earthly-docker`) and published image/repo names. Those change
+  only as part of the rename itself.
+
 ## Guidelines
 
 - Any code changes must be kept to a minimum to reduce the potential for merge conflicts with upstream `moby/buildkit`.
+- **Upstream Fixes & Version Bumping:** If upstream `moby/buildkit` has already resolved an issue in a newer version, suggest bumping BuildKit instead of implementing a temporary local fix.
+- **Double Export & Output Duplication:** Do not patch BuildKit to deduplicate or suppress progress logs across concurrent solves or intermediate exports (`gatewayClient.Export`). BuildKit streams vertex output to all attached callers; preventing redundant solve requests belongs in EarthBuild (`earthfile2llb/converter.go`).

Dockerfile

@@ -329,7 +329,20 @@ ARG TARGETPLATFORM
 RUN --mount=target=/root/.cache,type=cache <<EOT
   set -ex
   xx-go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}"
-  xx-go install "github.com/wadey/gocovmerge@latest"
+  # NOTE for future upstream merges:
+  # Upstream uses: xx-go install "github.com/wadey/gocovmerge@latest"
+  # However, gocovmerge has no go.mod and unpinned dependencies. Under Go 1.21,
+  # 'go install ...@latest' dynamically resolves transitive dependency golang.org/x/tools/cover
+  # to the latest version (v0.51.0+), which requires go >= 1.26.0 and fails the build.
+  # When upstream upgrades the builder to Go >= 1.26 (or pins gocovmerge),
+  # this temporary module workaround can be reverted back to upstream's single xx-go install line.
+  mkdir -p /tmp/gocovmerge && cd /tmp/gocovmerge
+  go mod init gocovmerge
+  go get golang.org/x/tools@v0.21.0
+  go get github.com/wadey/gocovmerge@b5bfa59ec0ad
+  xx-go install github.com/wadey/gocovmerge
+  cd /
+  rm -rf /tmp/gocovmerge
   mkdir /out
   if ! xx-info is-cross; then
     /go/bin/gotestsum --version
@@ -419,7 +432,11 @@ ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz
 ENV BUILDKIT_SETUP_CGROUPV2_ROOT=1
 ENV CGO_ENABLED=0
 ENV GOTESTSUM_FORMAT=standard-verbose
-COPY --link --from=gotestsum /out/gotestsum /usr/bin/
+# NOTE for future upstream merges:
+# Upstream copies all test helpers (/out) to /usr/bin/ because it includes
+# gotestsum, gocovmerge, and the gotestsumandcover runner script.
+# Do NOT change this to /out/gotestsum or gotestsumandcover will be missing at test runtime.
+COPY --link --from=gotestsum /out /usr/bin/
 COPY --link --from=silo /usr/bin/silo /usr/bin/minio
 COPY --link --from=silo /usr/bin/mcli /usr/bin/mc
 COPY --link --from=nydus /out/nydus-static/* /usr/bin/

api/services/registry/proxy_test.go

@@ -31,8 +31,8 @@ var errUnexpectedMessage = errors.New("stream carried a message that was not a B
 func tunnel(t *testing.T, reg *httptest.Server) (*http.Client, func() int) {
 	t.Helper()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	t.Cleanup(cancel)
+	ctx, cancel := context.WithCancelCause(context.Background())
+	t.Cleanup(func() { cancel(errors.WithStack(context.Canceled)) })
 
 	srv := NewServer(reg.Listener.Addr().String())
 
@@ -301,7 +301,11 @@ func TestProxyServesAResponse(t *testing.T) {
 
 	client, _ := tunnel(t, reg)
 
-	resp, err := client.Get("http://registry.invalid/v2/img/blobs/sha256:0")
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://registry.invalid/v2/img/blobs/sha256:0", nil)
+	if err != nil {
+		t.Fatalf("creating request: %v", err)
+	}
+	resp, err := client.Do(req)
 	if err != nil {
 		t.Fatalf("GET through the proxy: %v", err)
 	}
@@ -341,7 +345,11 @@ func TestProxyDoesNotTruncateAResponseThatStallsMidBody(t *testing.T) {
 
 	client, _ := tunnel(t, reg)
 
-	resp, err := client.Get("http://registry.invalid/v2/img/blobs/sha256:0")
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://registry.invalid/v2/img/blobs/sha256:0", nil)
+	if err != nil {
+		t.Fatalf("creating request: %v", err)
+	}
+	resp, err := client.Do(req)
 	if err != nil {
 		t.Fatalf("GET through the proxy: %v", err)
 	}
@@ -375,7 +383,11 @@ func TestProxyReusesOneConnectionForTwoRequests(t *testing.T) {
 			time.Sleep(200 * time.Millisecond)
 		}
 
-		resp, err := client.Get("http://registry.invalid/v2/img/manifests/latest")
+		req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://registry.invalid/v2/img/manifests/latest", nil)
+		if err != nil {
+			t.Fatalf("creating request %d: %v", i+1, err)
+		}
+		resp, err := client.Do(req)
 		if err != nil {
 			t.Fatalf("request %d through the proxy: %v", i+1, err)
 		}
@@ -422,7 +434,12 @@ func TestProxyPassesALargeRequestBodyThrough(t *testing.T) {
 
 	client, _ := tunnel(t, reg)
 
-	resp, err := client.Post("http://registry.invalid/v2/img/blobs/uploads/", "application/octet-stream", bytes.NewReader(sent))
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://registry.invalid/v2/img/blobs/uploads/", bytes.NewReader(sent))
+	if err != nil {
+		t.Fatalf("creating request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	resp, err := client.Do(req)
 	if err != nil {
 		t.Fatalf("POST through the proxy: %v", err)
 	}

cache/remotecache/v1/chains.go

@@ -123,8 +123,6 @@ type DescriptorProviderPair struct {
 	InfoProvider content.InfoProvider
 }
 
-var _ withCheckDescriptor = DescriptorProviderPair{}
-
 func (p DescriptorProviderPair) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
 	return p.Provider.ReaderAt(ctx, desc)
 }
@@ -162,13 +160,6 @@ func (p DescriptorProviderPair) SnapshotLabels(descs []ocispecs.Descriptor, inde
 	return nil
 }
 
-func (p DescriptorProviderPair) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
-	if cd, ok := p.Provider.(withCheckDescriptor); ok {
-		return cd.CheckDescriptor(ctx, desc)
-	}
-	return nil
-}
-
 // item is an implementation of a record in the cache chain. After validation,
 // normalization and marshalling into the cache config, the item results form
 // into the "layers", while the digests and the links form into the "records".

cache/remotecache/v1/utils.go

@@ -11,11 +11,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-type withCheckDescriptor interface {
-	// CheckDescriptor is additional method on Provider to check if the descriptor is available without opening the reader
-	CheckDescriptor(context.Context, ocispecs.Descriptor) error
-}
-
 // sortConfig sorts the config structure to make sure it is deterministic
 func sortConfig(cc *CacheConfig) {
 	type indexedLayer struct {
@@ -283,11 +278,7 @@ func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) s
 		return ""
 	}
 
-<<<<<<< HEAD
-	if cd, ok := r.Provider.(withCheckDescriptor); ok && len(r.Descriptors) > 0 {
-=======
 	if r.Provider != nil {
->>>>>>> v0.14.1
 		for _, d := range r.Descriptors {
 			if _, err := r.Provider.Info(ctx, d.Digest); err != nil {
 				return ""

client/client.go

@@ -44,12 +44,12 @@ type ClientOpt interface {
 // New returns a new buildkit client. Address can be empty for the system-default address.
 func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error) {
 	gopts := []grpc.DialOption{
-		grpc.WithInitialWindowSize(65535 * 32),     //earthly
-		grpc.WithInitialConnWindowSize(65535 * 16), //earthly
+		grpc.WithInitialWindowSize(65535 * 32),     // earthly
+		grpc.WithInitialConnWindowSize(65535 * 16), // earthly
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
-		grpc.WithDefaultCallOptions(grpc_retry.WithMax(8)),                                                                     //earthly
-		grpc.WithDefaultCallOptions(grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(10*time.Millisecond, 0.1))), //earthly
+		grpc.WithDefaultCallOptions(grpc_retry.WithMax(8)),                                                                     // earthly
+		grpc.WithDefaultCallOptions(grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(10*time.Millisecond, 0.1))), // earthly
 	}
 	needDialer := true
 

client/client_test.go

@@ -167,10 +167,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testRelativeMountpoint,
 	testLocalSourceDiffer,
 	testLocalSourceWithHardlinksFilter,
-<<<<<<< HEAD
-=======
 	testNoTarOCIIndexMediaType,
->>>>>>> v0.14.1
 	testOCILayoutSource,
 	testOCILayoutPlatformSource,
 	testBuildExportZstd,

client/llb/exec.go

@@ -389,7 +389,7 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
 				Size_: m.tmpfsOpt.Size,
 			}
 		}
-		if m.hostBind { //earthly
+		if m.hostBind { // earthly
 			pm.MountType = pb.MountType_HOST_BIND
 		}
 		peo.Mounts = append(peo.Mounts, pm)

client/solve.go

@@ -57,25 +57,17 @@ type SolveOpt struct {
 type ExportEntry struct {
 	Type               string
 	Attrs              map[string]string
-<<<<<<< HEAD
-	Output             filesync.FileOutputFunc                 // for ExporterOCI and ExporterDocker
+	Output             filesync.FileOutputFunc                 // for ExporterOCI, ExporterDocker and ExporterEarthly
 	OutputDir          string                                  // for ExporterLocal
 	OutputDirFunc      func(map[string]string) (string, error) // for ExporterEarthly
 	OutputPullCallback pullping.PullCallback                   // for ExporterEarthly
-	VerboseProgressCB  fsutil.VerboseProgressCB
-=======
-	Output             func(map[string]string) (io.WriteCloser, error) // for ExporterOCI, ExporterDocker and ExporterEarthly
-	OutputDir          string                                          // for ExporterLocal
-	OutputDirFunc      func(map[string]string) (string, error)         // for ExporterEarthly
-	OutputPullCallback pullping.PullCallback                           // for ExporterEarthly
 	// OnReceiveProgress is called periodically as files/artifacts are received from the daemon
 	// via diffcopy (e.g. during artifact export in EarthBuild). It reports the cumulative bytes received
 	// and whether the transfer is complete (done=true), allowing the client to render progress bars
 	// and transfer metrics.
 	OnReceiveProgress func(bytes int, done bool) // for ExporterEarthly
 	// OnReceiveFile is called for each file received from the daemon during export
 	OnReceiveFile func(path string, size int64) // for ExporterEarthly
->>>>>>> origin/main
 }
 
 type CacheOptionsEntry struct {
@@ -171,54 +163,10 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			contentStores[key2] = store
 		}
 
-<<<<<<< HEAD
 		var syncTargets []filesync.FSSyncTarget
 		for exID, ex := range opt.Exports {
 			var supportFile bool
 			var supportDir bool
-=======
-		var supportFile bool
-		var supportDir bool
-		switch ex.Type {
-		case ExporterLocal:
-			supportDir = true
-		case ExporterTar:
-			supportFile = true
-		case ExporterOCI, ExporterDocker:
-			supportDir = ex.OutputDir != ""
-			supportFile = ex.Output != nil
-		case ExporterEarthly:
-			supportFile = true
-		}
-
-		if supportFile && supportDir {
-			return nil, errors.Errorf("both file and directory output is not supported by %s exporter", ex.Type)
-		}
-		if !supportFile && ex.Output != nil {
-			return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
-		}
-		if !supportDir && ex.OutputDir != "" {
-			return nil, errors.Errorf("output directory is not supported by %s exporter", ex.Type)
-		}
-
-		if supportFile {
-			if ex.Output == nil {
-				return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type)
-			}
-			if ex.Type == ExporterEarthly {
-				if ex.OutputPullCallback != nil {
-					s.Allow(pullping.NewPullPing(ex.OutputPullCallback))
-				}
-				s.Allow(filesync.NewFSSyncMultiTarget(ex.Output, ex.OutputDirFunc, ex.OnReceiveProgress, ex.OnReceiveFile))
-			} else {
-				s.Allow(filesync.NewFSSyncTarget(ex.Output))
-			}
-		}
-		if supportDir {
-			if ex.OutputDir == "" {
-				return nil, errors.Errorf("output directory is required for %s exporter", ex.Type)
-			}
->>>>>>> origin/main
 			switch ex.Type {
 			case ExporterLocal:
 				supportDir = true
@@ -247,7 +195,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 					if ex.OutputPullCallback != nil {
 						s.Allow(pullping.NewPullPing(ex.OutputPullCallback))
 					}
-					s.Allow(filesync.NewFSSyncMultiTarget(ex.Output, ex.OutputDirFunc, ex.VerboseProgressCB))
+					s.Allow(filesync.NewFSSyncMultiTarget(ex.Output, ex.OutputDirFunc, ex.OnReceiveProgress, ex.OnReceiveFile))
 				} else {
 					syncTargets = append(syncTargets, filesync.WithFSSync(exID, ex.Output))
 				}
@@ -270,13 +218,6 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 				default:
 					syncTargets = append(syncTargets, filesync.WithFSSyncDir(exID, ex.OutputDir))
 				}
-<<<<<<< HEAD
-=======
-				contentStores["export"] = cs
-				storesToUpdate = append(storesToUpdate, ex.OutputDir)
-			default:
-				s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
->>>>>>> origin/main
 			}
 		}
 

cmd/buildkitd/main.go

@@ -16,7 +16,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/pkg/seed" //nolint:staticcheck // SA1019 deprecated
 	"github.com/containerd/containerd/pkg/userns"
 	"github.com/containerd/containerd/platforms"
@@ -25,12 +24,8 @@ import (
 	sddaemon "github.com/coreos/go-systemd/v22/daemon"
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/gofrs/flock"
-<<<<<<< HEAD
-	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
-	regproxy "github.com/moby/buildkit/api/services/registry"
-=======
 	"github.com/hashicorp/go-multierror"
->>>>>>> v0.14.1
+	regproxy "github.com/moby/buildkit/api/services/registry"
 	"github.com/moby/buildkit/cache/remotecache"
 	"github.com/moby/buildkit/cache/remotecache/azblob"
 	"github.com/moby/buildkit/cache/remotecache/gha"
@@ -270,15 +265,13 @@ func main() {
 			logrus.SetLevel(logrus.TraceLevel)
 		}
 
-<<<<<<< HEAD
 		logrus.SetOutput(os.Stderr) // earthly-specific: force logs to show up under earthly-buildkitd container logs
-=======
+
 		if sc := cfg.System; sc != nil {
 			if v := sc.PlatformsCacheMaxAge; v != nil {
 				archutil.CacheMaxAge = v.Duration
 			}
 		}
->>>>>>> v0.14.1
 
 		if cfg.GRPC.DebugAddress != "" {
 			if err := setupDebugHandlers(cfg.GRPC.DebugAddress); err != nil {
@@ -303,29 +296,15 @@ func main() {
 			otelgrpc.WithMeterProvider(mp),
 			otelgrpc.WithPropagators(propagators),
 		)
-<<<<<<< HEAD
-
-		unary := grpc_middleware.ChainUnaryServer(unaryInterceptor(ctx, tp, mp), grpcerrors.UnaryServerInterceptor,
-			unaryTimeoutInterceptor(), // earthly-specific
-		)
-		stream := grpc_middleware.ChainStreamServer(streamTracer, grpcerrors.StreamServerInterceptor,
-			streamTimeoutInterceptor(), // earthly-specific
-		)
-
 		maxMsgSize := 67108864 // 64MB
 		opts := []grpc.ServerOption{
-			grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream),
-			grpc.MaxRecvMsgSize(maxMsgSize), grpc.MaxSendMsgSize(maxMsgSize),
+			grpc.StatsHandler(statsHandler),
+			grpc.ChainUnaryInterceptor(unaryInterceptor, grpcerrors.UnaryServerInterceptor, unaryTimeoutInterceptor()),
+			grpc.ChainStreamInterceptor(grpcerrors.StreamServerInterceptor, streamTimeoutInterceptor()),
+			grpc.MaxRecvMsgSize(maxMsgSize),
+			grpc.MaxSendMsgSize(maxMsgSize),
 			grpc.InitialWindowSize(65535 * 32),
 			grpc.InitialConnWindowSize(65535 * 16),
-=======
-		opts := []grpc.ServerOption{
-			grpc.StatsHandler(statsHandler),
-			grpc.ChainUnaryInterceptor(unaryInterceptor, grpcerrors.UnaryServerInterceptor),
-			grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor),
-			grpc.MaxRecvMsgSize(defaults.DefaultMaxRecvMsgSize),
-			grpc.MaxSendMsgSize(defaults.DefaultMaxSendMsgSize),
->>>>>>> v0.14.1
 		}
 		server := grpc.NewServer(opts...)