diff --git a/cmd/agent/daemon.go b/cmd/agent/daemon.go index b66fca867..05abb9096 100644 --- a/cmd/agent/daemon.go +++ b/cmd/agent/daemon.go @@ -229,6 +229,8 @@ func runDaemon(root string) { // Step 6: bind the Unix socket. The run directory (0700) is already // ensured above; explicit socket chmod 0600 guards against any umask. + // The socket trusts callers running as this user: they can already read + // the user's files, so the daemon requires no separate connection token. ln, err := net.Listen("unix", paths.Sock) if err != nil { agentLogger.Error("daemon listen failed", "path", paths.Sock, "err", err) @@ -314,6 +316,7 @@ func runDaemon(root string) { pty.Close() // hang up every terminal, as closing its tab does chat.Close() // and end every Claude tab's program lsp.Close() + fsys.DropUploads(nil) if hooksrv != nil { _ = hooksrv.Close() } @@ -391,6 +394,7 @@ func runDaemon(root string) { var preempted atomic.Int32 host := stdioserver.New(d, conn, conn, agentLogger) + host.RegisterShutdownHook(func() { fsys.DropUploads(host.Context()) }) // Ready goes first: the client takes the first frame it gets for the // handshake, and an event ahead of it reads as an agent without an diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 542afbdb7..0b93c4799 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -266,6 +266,9 @@ func newHookGetInfoHandler(hs hookInfoProvider) dispatch.Handler { if err := hs.WaitReady(waitCtx); err != nil { return nil, proto.NewError(proto.CodeUnavailable, "hookserver not ready: "+err.Error()) } + // The daemon's 0600 socket trusts this user, whose processes can + // already read the user's files. The master hook token can be returned + // over that connection without a separate connection token. return map[string]any{ "socketPath": hs.SocketPath(), "token": hs.Token(), diff --git a/docs/CLI.md b/docs/CLI.md index 899a0491d..c405f9a16 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -208,8 +208,9 @@ nexus file push --device "Mac mini" --workspace site -r ./snapshot /Users/me/Dow ``` - `pull` 에서 로컬 대상을 생략하거나 기존 폴더를 지정하면 원래 이름으로 저장합니다. `push` 에서 원격 대상이 기존 폴더이면 그 안에 원래 이름으로 넣습니다. 여러 소스를 보내려면 대상이 기존 폴더여야 합니다. 각 파일의 부모 폴더가 있어야 하며, `-r` 은 복사할 폴더 트리를 만듭니다. -- 워크스페이스 안 읽기에는 파일 공유 권한이 필요합니다. 워크스페이스 안 쓰기에는 채팅 제어와 에이전트 요청을 허용해야 합니다. 워크스페이스 밖 절대 경로를 읽거나 쓰려면 대상 컴퓨터에서 터미널 제어와 에이전트 요청을 허용해야 합니다. 에이전트 설정이 ‘묻기’이면 쓰기와 워크스페이스 밖 읽기를 승인해야 합니다. 한 번 승인하면 같은 복사의 파일 조각과 승인된 폴더의 하위 경로에 적용됩니다. `push` 와 워크스페이스 밖 전송에는 대상 앱의 원격 프로토콜 0.14.0 이상과 로컬 워크스페이스가 필요합니다. +- 워크스페이스 안 읽기에는 파일 공유 권한이 필요합니다. 워크스페이스 안 쓰기에는 채팅 제어와 에이전트 요청을 허용해야 합니다. 워크스페이스 밖 절대 경로를 읽거나 쓰려면 대상 컴퓨터에서 터미널 제어와 에이전트 요청을 허용해야 합니다. 에이전트 설정이 ‘묻기’이면 쓰기와 워크스페이스 밖 읽기를 승인해야 합니다. 한 번 승인하면 같은 복사의 파일 조각과 승인된 폴더의 하위 경로에 적용됩니다. `push` 와 워크스페이스 밖 전송에는 대상 앱의 원격 프로토콜 0.14.0 이상이 필요합니다. SSH·WSL 워크스페이스로 보내려면 대상 파일 에이전트가 업로드를 지원해야 합니다. 워크스페이스 밖 절대 경로는 대상 앱의 로컬 워크스페이스에서 사용할 수 있습니다. - 각 파일은 전송을 마친 뒤 대상에 배치되며 기존 파일을 덮어씁니다. `push` 는 파일 하나당 최대 20 GiB 를 보낼 수 있으며 기존 대상 파일의 권한 비트를 유지합니다. `-r` 은 심볼릭 링크를 건너뜁니다. `pull` 은 원격 폴더 목록이 잘렸으면 복사를 시작하기 전에 실패합니다. +- Windows에서 `push`로 파일을 덮어쓰면 기존 파일의 ACL과 읽기 전용·숨김 속성은 유지되지 않습니다. - 출력에는 저장 경로, 바이트 수, SHA-256 이 들어갑니다. `--json` 은 기기와 두 컴퓨터의 경로도 출력합니다. `-r` 또는 여러 소스를 지정한 `push` 의 JSON 결과는 배열이며 건너뛴 링크에는 `"skipped": "symlink"` 가 붙습니다. ## 세션의 에이전트 따라가기 diff --git a/internal/fs/preview_unix.go b/internal/fs/preview_unix.go index 5a077e14a..ddc9adab6 100644 --- a/internal/fs/preview_unix.go +++ b/internal/fs/preview_unix.go @@ -13,15 +13,22 @@ import ( // Open the resolved absolute path one component at a time. O_NOFOLLOW // protects each component if it changes after the grant's symlink checks. func openPreviewFile(real string) (*os.File, error) { + return openCanonicalPath(real, false) +} + +func openCanonicalPath(real string, directory bool) (*os.File, error) { const flags = unix.O_CLOEXEC | unix.O_NOFOLLOW fd, err := unix.Open("/", flags|previewDirOpenFlags, 0) if err != nil { return nil, err } + if real == "/" && directory { + return os.NewFile(uintptr(fd), real), nil + } parts := strings.Split(strings.TrimPrefix(real, "/"), "/") for i, part := range parts { mode := flags | unix.O_RDONLY | unix.O_NONBLOCK - if i < len(parts)-1 { + if directory || i < len(parts)-1 { mode = flags | previewDirOpenFlags } next, err := unix.Openat(fd, part, mode, 0) diff --git a/internal/fs/service.go b/internal/fs/service.go index 04f8d3afd..8a44569b9 100644 --- a/internal/fs/service.go +++ b/internal/fs/service.go @@ -40,6 +40,10 @@ type EventSink func(event string, payload any) error type Service struct { root string + uploadMu sync.Mutex + uploads map[string]*fileUpload + uploadClosed bool + // watchMu serializes fs.watch, fs.unwatch and Close, so fsnotify Add and // Remove run in call order without holding mu, which the event loop // needs before it can take the next event from fsnotify. @@ -63,6 +67,7 @@ func New(root string) (*Service, error) { } return &Service{ root: filepath.Clean(abs), + uploads: make(map[string]*fileUpload), watches: make(map[string]*watchEntry), buffers: make(map[string]filewatch.FsChangeKind), }, nil @@ -75,8 +80,9 @@ func (s *Service) SetEventSink(sink EventSink) { s.sink = sink } -// Close stops watcher goroutines and clears buffered fs events. +// Close discards uploads, stops watchers, and clears buffered fs events. func (s *Service) Close() { + s.DropUploads(nil) s.watchMu.Lock() defer s.watchMu.Unlock() s.mu.Lock() @@ -108,6 +114,13 @@ func Register(d *dispatch.Dispatcher, fsys *Service) { d.Register("fs.readAbsoluteRange", fsys.ReadAbsoluteRange) d.Register("fs.readPreview", fsys.ReadPreview) d.Register("fs.writeFile", fsys.WriteFile) + d.Register("fs.upload.resolve", bindUpload(fsys.resolveUpload)) + d.Register("fs.upload.mkdir", bindUpload(fsys.mkdirUpload)) + d.Register("fs.upload.begin", bindUpload(fsys.beginUpload)) + d.Register("fs.upload.chunk", bindUpload(fsys.chunkUpload)) + d.Register("fs.upload.verify", bindUpload(fsys.verifyUpload)) + d.Register("fs.upload.commit", bindUpload(fsys.commitUpload)) + d.Register("fs.upload.abort", bindUpload(fsys.abortUpload)) d.Register("fs.createFile", fsys.CreateFile) d.Register("fs.mkdir", fsys.Mkdir) d.Register("fs.unlink", fsys.Unlink) diff --git a/internal/fs/upload.go b/internal/fs/upload.go new file mode 100644 index 000000000..fe78933f8 --- /dev/null +++ b/internal/fs/upload.go @@ -0,0 +1,516 @@ +package fs + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/nexus-code/nexus-code/internal/proto" +) + +const ( + maxUploads = 8 + maxUploadSize = 20 * 1024 * 1024 * 1024 + maxUploadChunk = 512 * 1024 + uploadIdle = 2 * time.Minute + staleUploadAge = 24 * time.Hour + uploadPrefix = ".nexus-push-" + uploadSuffix = ".tmp" +) + +var uploadIDPattern = regexp.MustCompile(`^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$`) + +type uploadParams struct { + RelPath string `json:"relPath"` + ExpectedPath string `json:"expectedPath"` + Directory bool `json:"directory"` + UploadID string `json:"uploadId"` + Size int64 `json:"size"` + Offset int64 `json:"offset"` + Data string `json:"data"` + SHA256 string `json:"sha256"` +} + +type uploadDestination struct { + path, parent string + info os.FileInfo +} + +type uploadSpace struct { + volume uint64 + free uint64 +} + +type fileUpload struct { + mu sync.Mutex + owner context.Context + ctx context.Context + cancel context.CancelFunc + stop func() bool + timer *time.Timer + deadline time.Time + + relPath string + dest uploadDestination + dir *os.File + file *os.File + temp string + size int64 + received int64 + verified bool + + // Guarded by Service.uploadMu. + space *uploadSpace + reserved uint64 +} + +func (s *Service) uploadDestination(rel string, directory bool) (uploadDestination, error) { + var dest uploadDestination + local := filepath.FromSlash(rel) + if strings.ContainsRune(local, 0) || (local != "" && !filepath.IsLocal(local)) { + return dest, FSError{Code: CodeOutOfWorkspace, Path: rel} + } + asked, err := s.Resolve(local) + if err != nil { + return dest, err + } + relative, err := filepath.Rel(s.root, asked) + if err != nil { + return dest, err + } + if relative == "." && !directory { + return dest, proto.ProtocolError("the destination is a folder") + } + base, err := filepath.EvalSymlinks(s.root) + if err != nil { + return dest, mapPathError(err, s.root) + } + parent := base + if relative != "." { + parts := strings.Split(relative, string(filepath.Separator)) + for _, part := range parts[:len(parts)-1] { + parent = filepath.Join(parent, part) + info, err := os.Lstat(parent) + if err != nil { + return dest, mapPathError(err, parent) + } + if info.Mode()&os.ModeSymlink != 0 { + return dest, FSError{Code: CodeOutOfWorkspace, Path: "the path goes through a symbolic link"} + } + if !info.IsDir() { + return dest, FSError{Code: CodeNotDirectory, Path: parent} + } + } + } + parent, err = filepath.EvalSymlinks(parent) + if err != nil { + return dest, mapPathError(err, parent) + } + within, err := filepath.Rel(base, parent) + if err != nil || !filepath.IsLocal(within) { + return dest, FSError{Code: CodeOutOfWorkspace, Path: "the path leads outside the workspace's folder"} + } + info, err := os.Stat(parent) + if err != nil { + return dest, mapPathError(err, parent) + } + if !info.IsDir() { + return dest, FSError{Code: CodeNotDirectory, Path: parent} + } + target := parent + if relative != "." { + target = filepath.Join(parent, filepath.Base(asked)) + } + return uploadDestination{path: target, parent: parent, info: info}, nil +} + +func (s *Service) resolveUpload(_ context.Context, p uploadParams) (any, error) { + dest, err := s.uploadDestination(p.RelPath, p.Directory) + return map[string]string{"path": dest.path}, err +} + +func changedUploadDestination() error { + return FSError{Code: CodeOutOfWorkspace, Path: "the destination folder changed during the upload"} +} + +func (dest uploadDestination) openParent(expected string) (*os.File, error) { + if dest.path != expected { + return nil, changedUploadDestination() + } + dir, err := openUploadParent(dest.parent) + if err != nil { + return nil, mapWriteError(err, dest.parent) + } + info, err := dir.Stat() + if err != nil || !os.SameFile(info, dest.info) { + _ = dir.Close() + return nil, changedUploadDestination() + } + return dir, nil +} + +func (s *Service) mkdirUpload(ctx context.Context, p uploadParams) (any, error) { + dest, err := s.uploadDestination(p.RelPath, true) + if err != nil { + return nil, err + } + dir, err := dest.openParent(p.ExpectedPath) + if err != nil { + return nil, err + } + defer dir.Close() + if err := ctx.Err(); err != nil { + return nil, err + } + if dest.path == dest.parent { + return struct{}{}, nil + } + err = mkdirUploadAt(dir, filepath.Base(dest.path)) + if errors.Is(err, os.ErrExist) { + _, err = uploadTarget(dest.path, true) + } + if err != nil { + return nil, mapWriteError(err, dest.path) + } + return struct{}{}, nil +} + +func uploadTarget(path string, directory bool) (os.FileInfo, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) && !directory { + return nil, nil + } + if err != nil { + return nil, mapWriteError(err, path) + } + if info.Mode()&os.ModeSymlink != 0 { + return nil, FSError{Code: CodeOutOfWorkspace, Path: "the destination is a symbolic link"} + } + if directory && !info.IsDir() { + return nil, proto.ProtocolError("the destination is not a folder") + } + if !directory && !info.Mode().IsRegular() { + return nil, proto.ProtocolError("the destination is not a regular file") + } + return info, nil +} + +func (s *Service) beginUpload(ctx context.Context, p uploadParams) (_ any, err error) { + if !uploadIDPattern.MatchString(p.UploadID) || p.Size < 0 || p.Size > maxUploadSize { + return nil, proto.ProtocolError("invalid upload ID or size") + } + s.uploadMu.Lock() + if s.uploadClosed { + s.uploadMu.Unlock() + return nil, proto.NewError(proto.CodeUnavailable, "the filesystem agent is shutting down") + } + if len(s.uploads) >= maxUploads { + s.uploadMu.Unlock() + return nil, proto.NewError(proto.CodeUnavailable, "the host is busy writing files") + } + if s.uploads[p.UploadID] != nil { + s.uploadMu.Unlock() + return nil, proto.ProtocolError("the upload ID is already in use") + } + uploadCtx, cancel := context.WithCancel(ctx) + u := &fileUpload{owner: ctx, ctx: uploadCtx, cancel: cancel, relPath: p.RelPath, size: p.Size} + u.mu.Lock() + s.uploads[p.UploadID] = u + s.uploadMu.Unlock() + defer u.mu.Unlock() + defer func() { + if err != nil { + s.removeUpload(p.UploadID, u) + } + }() + u.stop = context.AfterFunc(ctx, func() { s.abortPendingUpload(p.UploadID, u) }) + u.dest, err = s.uploadDestination(p.RelPath, false) + if err != nil { + return nil, err + } + u.dir, err = u.dest.openParent(p.ExpectedPath) + if err != nil { + return nil, err + } + if _, err = uploadTarget(u.dest.path, false); err != nil { + return nil, err + } + removeStaleUploads(u.ctx, u.dir, filepath.Base(u.dest.path)) + if err = s.reserveUpload(u); err != nil { + return nil, err + } + if err = u.ctx.Err(); err != nil { + return nil, err + } + name := uploadPrefix + p.UploadID + uploadSuffix + u.file, err = createUploadAt(u.dir, name) + if err != nil { + return nil, mapWriteError(err, u.dest.path) + } + u.temp = name + s.touchUpload(p.UploadID, u) + return struct{}{}, nil +} + +// Reap only regular files under the generated name and older than a day. +func removeStaleUploads(ctx context.Context, dir *os.File, target string) { + listing, err := openUploadListing(dir) + if err != nil { + return + } + defer listing.Close() + cutoff := time.Now().Add(-staleUploadAge) + for ctx.Err() == nil { + names, err := listing.Readdirnames(64) + for _, name := range names { + id, hasPrefix := strings.CutPrefix(name, uploadPrefix) + id, hasSuffix := strings.CutSuffix(id, uploadSuffix) + if hasPrefix && hasSuffix && uploadIDPattern.MatchString(id) && !strings.EqualFold(name, target) { + removeStaleUploadAt(dir, name, cutoff) + } + } + if err != nil { + return + } + } +} + +func (s *Service) reserveUpload(u *fileUpload) error { + space, err := availableUploadSpace(u.dir) + if err != nil || space == nil { + return err + } + s.uploadMu.Lock() + defer s.uploadMu.Unlock() + needed := uint64(u.size) + for _, other := range s.uploads { + if other.space != nil && other.space.volume == space.volume { + needed += other.reserved + } + } + if needed > space.free { + return FSError{Code: CodeNoSpace, Path: u.dest.path} + } + u.space = space + u.reserved = uint64(u.size) + return nil +} + +func (s *Service) pendingUpload(ctx context.Context, id string) *fileUpload { + s.uploadMu.Lock() + defer s.uploadMu.Unlock() + if u := s.uploads[id]; u != nil && u.owner == ctx { + return u + } + return nil +} + +// withUpload serializes disk operations for one upload. The room bounds +// parallel pieces and checks their disjoint coverage before relaying them. +func (s *Service) withUpload(ctx context.Context, id string, run func(*fileUpload) error) (any, error) { + u := s.pendingUpload(ctx, id) + if u == nil { + return nil, FSError{Code: CodeNotFound, Path: "no such pending file upload"} + } + u.mu.Lock() + defer u.mu.Unlock() + if err := u.ctx.Err(); err != nil { + return nil, err + } + if err := run(u); err != nil { + s.removeUpload(id, u) + return nil, err + } + return struct{}{}, nil +} + +func (s *Service) chunkUpload(ctx context.Context, p uploadParams) (any, error) { + return s.withUpload(ctx, p.UploadID, func(u *fileUpload) error { + if u.verified || len(p.Data) > base64.StdEncoding.EncodedLen(maxUploadChunk) { + return proto.ProtocolError("the upload is not accepting this piece") + } + data, err := base64.StdEncoding.Strict().DecodeString(p.Data) + size := int64(len(data)) + if err != nil || size == 0 || size > maxUploadChunk || + base64.StdEncoding.EncodeToString(data) != p.Data { + return proto.ProtocolError("the piece has an invalid length or encoding") + } + if p.Offset < 0 || p.Offset > u.size-size || u.received > u.size-size { + return proto.ProtocolError("the piece exceeds the declared file size") + } + if _, err := u.file.WriteAt(data, p.Offset); err != nil { + return mapWriteError(err, u.dest.path) + } + u.received += size + if u.space != nil { + s.uploadMu.Lock() + u.reserved -= uint64(size) + s.uploadMu.Unlock() + } + s.touchUpload(p.UploadID, u) + return nil + }) +} + +// Verify before the desktop checks consent again and asks for the rename. +func (s *Service) verifyUpload(ctx context.Context, p uploadParams) (any, error) { + return s.withUpload(ctx, p.UploadID, func(u *fileUpload) error { + info, err := u.file.Stat() + if err != nil { + return err + } + if u.received != u.size || info.Size() != u.size { + return proto.ProtocolError("the upload did not receive all the declared bytes") + } + hash := sha256.New() + buf := make([]byte, maxUploadChunk) + for offset := int64(0); offset < u.size; { + if err := u.ctx.Err(); err != nil { + return err + } + n, err := u.file.ReadAt(buf[:min(u.size-offset, int64(len(buf)))], offset) + if err != nil { + return err + } + _, _ = hash.Write(buf[:n]) + offset += int64(n) + } + if hex.EncodeToString(hash.Sum(nil)) != p.SHA256 { + return proto.ProtocolError("the upload's SHA-256 did not match") + } + u.verified = true + s.touchUpload(p.UploadID, u) + return nil + }) +} + +func (s *Service) commitUpload(ctx context.Context, p uploadParams) (any, error) { + return s.withUpload(ctx, p.UploadID, func(u *fileUpload) error { + if !u.verified { + return proto.ProtocolError("the upload has not been verified") + } + dest, err := s.uploadDestination(u.relPath, false) + if err != nil { + return err + } + if dest.path != u.dest.path || !os.SameFile(dest.info, u.dest.info) { + return changedUploadDestination() + } + existing, err := uploadTarget(dest.path, false) + if err != nil { + return err + } + if !sameUploadAt(u.dir, u.temp, u.file) { + return FSError{Code: CodeOutOfWorkspace, Path: "the temporary file changed during the upload"} + } + if err := u.ctx.Err(); err != nil { + return err + } + if err := publishUploadAt(u.dir, u.temp, filepath.Base(dest.path), u.file, existing); err != nil { + return mapWriteError(err, dest.path) + } + u.temp = "" + s.removeUpload(p.UploadID, u) + return nil + }) +} + +func (s *Service) touchUpload(id string, u *fileUpload) { + u.deadline = time.Now().Add(uploadIdle) + if u.timer == nil { + u.timer = time.AfterFunc(uploadIdle, func() { + u.mu.Lock() + defer u.mu.Unlock() + if u.ctx.Err() != nil { + return + } + if remaining := time.Until(u.deadline); remaining > 0 { + u.timer.Reset(remaining) + } else { + s.removeUpload(id, u) + } + }) + } +} + +func (s *Service) removeUpload(id string, u *fileUpload) { + u.cancel() + if u.stop != nil { + u.stop() + } + if u.timer != nil { + u.timer.Stop() + } + if u.file != nil { + if u.temp != "" && sameUploadAt(u.dir, u.temp, u.file) { + if err := removeUploadAt(u.dir, u.temp); err != nil && !errors.Is(err, os.ErrNotExist) { + slog.Warn("could not remove temporary upload", "path", u.dest.path, "err", err) + } + } + _ = u.file.Close() + u.file = nil + } + if u.dir != nil { + _ = u.dir.Close() + u.dir = nil + } + s.uploadMu.Lock() + if s.uploads[id] == u { + delete(s.uploads, id) + } + s.uploadMu.Unlock() +} + +func (s *Service) abortPendingUpload(id string, u *fileUpload) { + u.cancel() + u.mu.Lock() + defer u.mu.Unlock() + s.removeUpload(id, u) +} + +func (s *Service) abortUpload(ctx context.Context, p uploadParams) (any, error) { + if u := s.pendingUpload(ctx, p.UploadID); u != nil { + s.abortPendingUpload(p.UploadID, u) + } + return struct{}{}, nil +} + +// DropUploads drains uploads owned by a closed connection, or all uploads +// when the service shuts down. Daemon reattach keeps the other services alive. +func (s *Service) DropUploads(owner context.Context) { + s.uploadMu.Lock() + if owner == nil { + s.uploadClosed = true + } + pending := make(map[string]*fileUpload) + for id, u := range s.uploads { + if owner == nil || u.owner == owner { + u.cancel() + pending[id] = u + } + } + s.uploadMu.Unlock() + for id, u := range pending { + s.abortPendingUpload(id, u) + } +} + +func bindUpload(handler func(context.Context, uploadParams) (any, error)) func(context.Context, json.RawMessage) (any, error) { + return func(ctx context.Context, raw json.RawMessage) (any, error) { + var p uploadParams + if err := json.Unmarshal(raw, &p); err != nil { + return nil, proto.ProtocolError("invalid upload parameters") + } + return handler(ctx, p) + } +} diff --git a/internal/fs/upload_other.go b/internal/fs/upload_other.go new file mode 100644 index 000000000..54eaa1e08 --- /dev/null +++ b/internal/fs/upload_other.go @@ -0,0 +1,53 @@ +//go:build !darwin && !linux + +package fs + +import ( + "os" + "path/filepath" + "time" +) + +func openUploadParent(path string) (*os.File, error) { + return os.Open(path) +} + +func openUploadListing(dir *os.File) (*os.File, error) { + return os.Open(dir.Name()) +} + +func removeStaleUploadAt(dir *os.File, name string, cutoff time.Time) { + info, err := os.Lstat(filepath.Join(dir.Name(), name)) + if err != nil || !info.Mode().IsRegular() || !info.ModTime().Before(cutoff) { + return + } + opened, err := dir.Stat() + if err != nil { + return + } + current, err := os.Stat(dir.Name()) + if err == nil && os.SameFile(opened, current) { + _ = removeUploadAt(dir, name) + } +} + +func availableUploadSpace(_ *os.File) (*uploadSpace, error) { + return nil, nil +} + +func mkdirUploadAt(dir *os.File, name string) error { + return os.Mkdir(filepath.Join(dir.Name(), name), 0o777) +} + +func sameUploadAt(dir *os.File, name string, file *os.File) bool { + opened, err := file.Stat() + if err != nil { + return false + } + named, err := os.Lstat(filepath.Join(dir.Name(), name)) + return err == nil && os.SameFile(opened, named) +} + +func removeUploadAt(dir *os.File, name string) error { + return os.Remove(filepath.Join(dir.Name(), name)) +} diff --git a/internal/fs/upload_portable.go b/internal/fs/upload_portable.go new file mode 100644 index 000000000..4df706484 --- /dev/null +++ b/internal/fs/upload_portable.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows + +package fs + +import ( + "os" + "path/filepath" +) + +func createUploadAt(dir *os.File, name string) (*os.File, error) { + return os.OpenFile(filepath.Join(dir.Name(), name), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) +} + +func publishUploadAt(dir *os.File, temp, target string, _ *os.File, _ os.FileInfo) error { + return os.Rename(filepath.Join(dir.Name(), temp), filepath.Join(dir.Name(), target)) +} diff --git a/internal/fs/upload_unix.go b/internal/fs/upload_unix.go new file mode 100644 index 000000000..53c7a9eb3 --- /dev/null +++ b/internal/fs/upload_unix.go @@ -0,0 +1,90 @@ +//go:build darwin || linux + +package fs + +import ( + "os" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +// Capture the inherited mask during package initialization, before requests +// can create files. Chmod at publish time must apply the mask explicitly. +var newUploadMode = func() os.FileMode { + mask := unix.Umask(0o777) + unix.Umask(mask) + return 0o666 &^ os.FileMode(mask) +}() + +func openUploadParent(path string) (*os.File, error) { + return openCanonicalPath(path, true) +} + +// The upload's parent handle only has search access. Open a listing +// relative to it so a path rename cannot redirect the sweep. +func openUploadListing(dir *os.File) (*os.File, error) { + fd, err := unix.Openat(int(dir.Fd()), ".", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), dir.Name()), nil +} + +func removeStaleUploadAt(dir *os.File, name string, cutoff time.Time) { + var info unix.Stat_t + if unix.Fstatat(int(dir.Fd()), name, &info, unix.AT_SYMLINK_NOFOLLOW) == nil && + info.Mode&unix.S_IFMT == unix.S_IFREG && + time.Unix(int64(info.Mtim.Sec), int64(info.Mtim.Nsec)).Before(cutoff) { + _ = removeUploadAt(dir, name) + } +} + +func availableUploadSpace(dir *os.File) (*uploadSpace, error) { + fd := int(dir.Fd()) + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return nil, err + } + var space unix.Statfs_t + if err := unix.Fstatfs(fd, &space); err != nil { + return nil, err + } + return &uploadSpace{volume: uint64(stat.Dev), free: uint64(space.Bavail) * uint64(space.Bsize)}, nil +} + +func createUploadAt(dir *os.File, name string) (*os.File, error) { + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_CREAT|unix.O_EXCL|unix.O_RDWR|unix.O_CLOEXEC, 0o600) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), filepath.Join(dir.Name(), name)), nil +} + +func mkdirUploadAt(dir *os.File, name string) error { + return unix.Mkdirat(int(dir.Fd()), name, 0o777) +} + +func sameUploadAt(dir *os.File, name string, file *os.File) bool { + var opened, named unix.Stat_t + return unix.Fstat(int(file.Fd()), &opened) == nil && + unix.Fstatat(int(dir.Fd()), name, &named, unix.AT_SYMLINK_NOFOLLOW) == nil && + opened.Dev == named.Dev && opened.Ino == named.Ino +} + +func removeUploadAt(dir *os.File, name string) error { + return unix.Unlinkat(int(dir.Fd()), name, 0) +} + +func publishUploadAt(dir *os.File, temp, target string, file *os.File, existing os.FileInfo) error { + mode := newUploadMode + if existing != nil { + mode = existing.Mode() & (os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky) + } + if err := file.Chmod(mode); err != nil { + return err + } + fd := int(dir.Fd()) + return unix.Renameat(fd, temp, fd, target) +} diff --git a/internal/fs/upload_windows.go b/internal/fs/upload_windows.go new file mode 100644 index 000000000..aaaf53689 --- /dev/null +++ b/internal/fs/upload_windows.go @@ -0,0 +1,31 @@ +package fs + +import ( + "os" + "path/filepath" + + "golang.org/x/sys/windows" +) + +func createUploadAt(dir *os.File, name string) (*os.File, error) { + path := filepath.Join(dir.Name(), name) + p, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + // Permit rename/unlink while holding the identity of the temporary file. + handle, err := windows.CreateFile(p, windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, windows.CREATE_NEW, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(handle), path), nil +} + +// Path-based operations assume openUploadParent's os.Open holds the parent +// without FILE_SHARE_DELETE, preventing its name from being swapped while open. +func publishUploadAt(dir *os.File, temp, target string, _ *os.File, _ os.FileInfo) error { + // Windows uses inherited ACLs and the Go rename replaces existing files. + return os.Rename(filepath.Join(dir.Name(), temp), filepath.Join(dir.Name(), target)) +} diff --git a/internal/fs/write_test.go b/internal/fs/write_test.go index 5444a280c..487dfea3b 100644 --- a/internal/fs/write_test.go +++ b/internal/fs/write_test.go @@ -2,6 +2,7 @@ package fs import ( "context" + "crypto/sha256" "encoding/json" "fmt" "os" @@ -9,6 +10,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/nexus-code/nexus-code/internal/dispatch" "github.com/nexus-code/nexus-code/internal/proto" @@ -713,3 +715,197 @@ func TestWriteFile_HardLinkWrittenInPlace(t *testing.T) { t.Fatalf("hard link content %q, want v2", got) } } + +func TestUploadPathEscape(t *testing.T) { + for _, kind := range []string{"relative", "absolute", "parent link", "destination link", "replaced parent"} { + t.Run(kind, func(t *testing.T) { + if runtime.GOOS == "windows" && strings.Contains(kind, "link") { + t.Skip("creating symlinks requires privileges on Windows") + } + if kind == "replaced parent" && runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("directory handles anchor the temporary file on Linux and macOS") + } + root, outside := t.TempDir(), t.TempDir() + s, err := New(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.Close) + ctx := context.Background() + p := uploadParams{UploadID: "a0000000-0000-4000-8000-000000000000"} + switch kind { + case "relative": + p.RelPath = "../outside" + case "absolute": + p.RelPath = filepath.Join(outside, "file") + case "parent link", "destination link": + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + p.RelPath = "link" + if kind == "parent link" { + p.RelPath += "/file" + } + case "replaced parent": + parent := filepath.Join(root, "nested") + moved := filepath.Join(root, "moved") + if err := os.Mkdir(parent, 0o700); err != nil { + t.Fatal(err) + } + p.RelPath = "nested/file" + dest, err := s.uploadDestination(p.RelPath, false) + if err != nil { + t.Fatal(err) + } + p.ExpectedPath = dest.path + if _, err := s.beginUpload(ctx, p); err != nil { + t.Fatal(err) + } + if err := os.Rename(parent, moved); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, parent); err != nil { + t.Fatal(err) + } + p.SHA256 = fmt.Sprintf("%x", sha256.Sum256(nil)) + if _, err := s.verifyUpload(ctx, p); err != nil { + t.Fatal(err) + } + _, err = s.commitUpload(ctx, p) + if proto.ErrorCode(err) != CodeOutOfWorkspace { + t.Fatalf("committing through a replaced parent: %v", err) + } + entries, err := os.ReadDir(moved) + if err != nil || len(entries) != 0 { + t.Fatalf("temporary file in moved directory: %v, %v", entries, err) + } + } + if kind != "replaced parent" { + dest, err := s.uploadDestination(p.RelPath, false) + if err == nil { + p.ExpectedPath = dest.path + _, err = s.beginUpload(ctx, p) + } + if proto.ErrorCode(err) != CodeOutOfWorkspace { + t.Fatalf("upload destination escaped: %v", err) + } + } + entries, err := os.ReadDir(outside) + if err != nil || len(entries) != 0 { + t.Fatalf("upload changed the outside directory: %v, %v", entries, err) + } + }) + } +} + +func TestUploadCleanup(t *testing.T) { + for _, kind := range []string{"verified commit", "hash mismatch", "abort", "disconnect"} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + s, err := New(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.Close) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := uploadParams{UploadID: "b0000000-0000-4000-8000-000000000000", RelPath: "data", Size: 3} + dest, err := s.uploadDestination(p.RelPath, false) + if err != nil { + t.Fatal(err) + } + p.ExpectedPath = dest.path + if err := os.WriteFile(dest.path, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := s.beginUpload(ctx, p); err != nil { + t.Fatal(err) + } + for _, piece := range []struct { + offset int64 + base64 string + }{{1, "ZXc="}, {0, "bg=="}} { + p.Offset, p.Data = piece.offset, piece.base64 + if _, err := s.chunkUpload(ctx, p); err != nil { + t.Fatal(err) + } + } + want := "old" + switch kind { + case "verified commit": + p.SHA256 = fmt.Sprintf("%x", sha256.Sum256([]byte("new"))) + if _, err := s.verifyUpload(ctx, p); err != nil { + t.Fatal(err) + } + if _, err := s.commitUpload(ctx, p); err != nil { + t.Fatal(err) + } + want = "new" + if runtime.GOOS != "windows" { + info, err := os.Stat(dest.path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("destination mode changed: %v, %v", info, err) + } + } + case "hash mismatch": + p.SHA256 = fmt.Sprintf("%x", sha256.Sum256([]byte("bad"))) + if _, err := s.verifyUpload(ctx, p); err == nil || !strings.Contains(err.Error(), "SHA-256 did not match") { + t.Fatalf("verification error: %v", err) + } + case "abort": + if _, err := s.abortUpload(ctx, p); err != nil { + t.Fatal(err) + } + case "disconnect": + cancel() + s.DropUploads(ctx) + } + if entries, err := os.ReadDir(root); err != nil || len(entries) != 1 { + t.Fatalf("temporary upload remains: %v, %v", entries, err) + } + if data, err := os.ReadFile(dest.path); err != nil || string(data) != want { + t.Fatalf("destination changed: %q, %v", data, err) + } + }) + } +} + +func TestBeginUploadRemovesStaleTemporaryFiles(t *testing.T) { + for _, test := range []struct { + kind, name string + kept bool + }{ + {"abandoned file", uploadPrefix + "a0000000-0000-4000-8000-000000000000" + uploadSuffix, false}, + {"recent file", uploadPrefix + "b0000000-0000-4000-8000-000000000000" + uploadSuffix, true}, + {"unrelated file", uploadPrefix + "not-an-upload" + uploadSuffix, true}, + {"matching directory", uploadPrefix + "c0000000-0000-4000-8000-000000000000" + uploadSuffix, true}, + } { + t.Run(test.kind, func(t *testing.T) { + root := t.TempDir() + entry := filepath.Join(root, test.name) + if test.kind == "matching directory" { + must(t, os.Mkdir(entry, 0o700)) + } else { + must(t, os.WriteFile(entry, nil, 0o600)) + } + if test.kind != "recent file" { + old := time.Now().Add(-25 * time.Hour) + must(t, os.Chtimes(entry, old, old)) + } + s, err := New(root) + must(t, err) + t.Cleanup(s.Close) + dest, err := s.uploadDestination("data", false) + must(t, err) + _, err = s.beginUpload(context.Background(), uploadParams{ + UploadID: "d0000000-0000-4000-8000-000000000000", + RelPath: "data", ExpectedPath: dest.path, + }) + must(t, err) + _, err = os.Lstat(entry) + if test.kept && err != nil || !test.kept && !os.IsNotExist(err) { + t.Fatalf("expected kept=%v, got stat error %v", test.kept, err) + } + }) + } +} diff --git a/internal/stdioserver/host.go b/internal/stdioserver/host.go index b096e6c07..a52c33a26 100644 --- a/internal/stdioserver/host.go +++ b/internal/stdioserver/host.go @@ -597,7 +597,7 @@ func (h *Host) beginRequest(method string, size int) (release func(), accepting } usage, countLimit, byteLimit := &h.regular, maxDispatchRequests, maxDispatchBytes switch method { - case "pty.ack", "search.cancel", "git.cancel", "lsp.cancel", "chat.stop", "lsp.shutdown": + case "pty.ack", "search.cancel", "git.cancel", "lsp.cancel", "chat.stop", "lsp.shutdown", "fs.upload.abort": usage, countLimit, byteLimit = &h.control, maxControlRequests, maxControlBytes } if usage.requests >= countLimit || size > byteLimit-usage.bytes { diff --git a/src/main/features/cli/remote.ts b/src/main/features/cli/remote.ts index f4f7d8635..12692b5e4 100644 --- a/src/main/features/cli/remote.ts +++ b/src/main/features/cli/remote.ts @@ -6,6 +6,7 @@ import { AGENT_CONSENT_MS, type AgentSender, + MAX_FILE_WRITE_BYTES, MAX_WAIT_MS, type RemoteSession, type RemoteWorkspace, @@ -14,7 +15,7 @@ import { type RpcParams, type RpcResult, } from "../../../../packages/remote-protocol/src"; -import { isFileTransferMethod, needsFileConsent } from "../remote/file-io"; +import { fileVerifyTimeoutMs, isFileTransferMethod, needsFileConsent } from "../remote/file-io"; import { type CliContext, type CliDevice, CliError, pick } from "./context"; import { ago } from "./format"; import { senderOf } from "./sender"; @@ -32,6 +33,8 @@ export const NOT_IN_ROOM = "this computer is not in the remote room (Settings const AGENT_REQUEST_MS = AGENT_CONSENT_MS + 10_000; /** How long a wait's request waits: its own time and the way there and back. */ const WAIT_REQUEST_MS = MAX_WAIT_MS + 5_000; +// The peer may need to verify a maximum-size upload and then commit it. +const FILE_WRITE_END_REQUEST_MS = fileVerifyTimeoutMs(MAX_FILE_WRITE_BYTES) + 35_000; /** A device as JSON names it beside what came from it. */ export function deviceRef(device: CliDevice): { id: string; name: string } { @@ -225,7 +228,7 @@ export async function askDevice( method === "chat.wait" || method === "terminal.wait" ? WAIT_REQUEST_MS : method === "files.write.end" - ? 5 * 60_000 + ? FILE_WRITE_END_REQUEST_MS : from && !reads && (!isFileTransferMethod(method) || diff --git a/src/main/features/remote/agent-file-writes.ts b/src/main/features/remote/agent-file-writes.ts new file mode 100644 index 000000000..1b616b33f --- /dev/null +++ b/src/main/features/remote/agent-file-writes.ts @@ -0,0 +1,89 @@ +import { z } from "zod"; +import { RpcErrorCode } from "../../../../packages/remote-protocol/src"; +import type { AgentCallOptions } from "../../infra/agent/channel"; +import { parseAgentResult } from "../fs/bridge/agent-provider"; +import type { AgentBackedProvider } from "../fs/bridge/provider"; +import { filePathRules, fileVerifyTimeoutMs, validFileName } from "./file-io"; +import { workspaceWritePath } from "./file-writes"; +import { RpcFailure } from "./room"; + +const pathResult = z.object({ path: z.string() }); + +export class AgentFileWrites { + private readonly relPath: string; + + constructor( + private readonly fs: AgentBackedProvider, + root: string, + path: string, + private readonly directory: boolean, + ) { + const rel = workspaceWritePath(root, path, directory); + if (rel && !rel.split("/").every((part) => validFileName(part, filePathRules(root)))) { + throw new RpcFailure(RpcErrorCode.InvalidParams, "the destination needs a file name"); + } + this.relPath = rel; + } + + async resolve(): Promise { + const result = await this.call("resolve", { relPath: this.relPath, directory: this.directory }); + return parseAgentResult(pathResult, result).path; + } + + async mkdir(expectedPath: string): Promise<{ path: string }> { + await this.call("mkdir", { relPath: this.relPath, expectedPath }); + return { path: expectedPath }; + } + + async begin(uploadId: string, expectedPath: string, size: number): Promise { + await this.call("begin", { relPath: this.relPath, expectedPath, uploadId, size }); + } + + async chunk(uploadId: string, offset: number, data: string): Promise { + await this.call("chunk", { uploadId, offset, data }); + } + + async verify(uploadId: string, sha256: string, size: number): Promise { + await this.call("verify", { uploadId, sha256 }, { timeoutMs: fileVerifyTimeoutMs(size) }); + } + + async commit(uploadId: string): Promise { + await this.call("commit", { uploadId }); + } + + async abort(uploadId: string): Promise { + await this.call("abort", { uploadId }); + } + + private async call(method: string, params: object, options?: AgentCallOptions): Promise { + try { + return await this.fs.callAgentMethod(`fs.upload.${method}`, params, options); + } catch (error) { + const code = (error as { code?: string } | null)?.code; + const message = error instanceof Error ? error.message : String(error); + switch (code) { + case "unsupported-method": + throw new RpcFailure( + RpcErrorCode.Unavailable, + "the remote filesystem agent must be updated to write files", + ); + case "OUT_OF_WORKSPACE": + case "PERMISSION_DENIED": + throw new RpcFailure(RpcErrorCode.Forbidden, message); + case "NOT_FOUND": + throw new RpcFailure(RpcErrorCode.NotFound, message); + case "IS_DIRECTORY": + case "NOT_DIRECTORY": + case "server.protocol-error": + throw new RpcFailure(RpcErrorCode.InvalidParams, message); + case "NO_SPACE": + throw new RpcFailure( + RpcErrorCode.Unavailable, + "the destination has insufficient disk space", + ); + default: + throw new RpcFailure(RpcErrorCode.Unavailable, message); + } + } + } +} diff --git a/src/main/features/remote/agent-gate.ts b/src/main/features/remote/agent-gate.ts index 455af0826..1a85e1fa0 100644 --- a/src/main/features/remote/agent-gate.ts +++ b/src/main/features/remote/agent-gate.ts @@ -9,7 +9,6 @@ * command's changes to another workspace are: for this once, or for every * change from that computer's workspace to this one until the app restarts. */ -import path from "node:path"; import { AGENT_CONSENT_MS, type AgentSender, @@ -22,6 +21,7 @@ import type { RelaySender } from "../../../shared/chat/relay"; import type { CliChange } from "../../../shared/cli/ui"; import type { RemoteAgentsPolicy } from "../../../shared/types/app-state"; import type { CliServices } from "../cli/context"; +import { filePathRules } from "./file-io"; import { RpcFailure } from "./room"; const MINUTE_MS = 60_000; @@ -66,10 +66,10 @@ interface FileGrant { } function covers(grant: FileGrant, target: string): boolean { + const { sep } = filePathRules(grant.path); return ( target === grant.path || - (grant.tree && - target.startsWith(grant.path.endsWith(path.sep) ? grant.path : grant.path + path.sep)) + (grant.tree && target.startsWith(grant.path.endsWith(sep) ? grant.path : grant.path + sep)) ); } diff --git a/src/main/features/remote/file-io.ts b/src/main/features/remote/file-io.ts index 686254f5d..b35ff0598 100644 --- a/src/main/features/remote/file-io.ts +++ b/src/main/features/remote/file-io.ts @@ -21,6 +21,11 @@ export function needsFileConsent(method: string, params: { anywhere?: true }): b export const FILE_WRITE_BUSY = "the host is busy writing files"; +/** Budget for a full re-read and hash at 32 MiB/s, plus request overhead. */ +export function fileVerifyTimeoutMs(size: number): number { + return 30_000 + Math.ceil((size * 1000) / (32 * 1024 ** 2)); +} + export function filePathRules(root: string): path.PlatformPath { return /^(?:[A-Za-z]:[\\/]|[\\/]{2}[^\\/])/.test(root) ? path.win32 : path.posix; } diff --git a/src/main/features/remote/file-writes.ts b/src/main/features/remote/file-writes.ts index af78d99d7..8abf25e79 100644 --- a/src/main/features/remote/file-writes.ts +++ b/src/main/features/remote/file-writes.ts @@ -5,6 +5,7 @@ import { lstat, mkdir, open, + opendir, realpath, rename, stat, @@ -19,6 +20,7 @@ import { type RpcResult, } from "../../../../packages/remote-protocol/src"; import { createLogger } from "../../../shared/log/main"; +import type { AgentFileWrites } from "./agent-file-writes"; import { FILE_WRITE_BUSY, readFileChunk, validFileName, writeFileChunk } from "./file-io"; import { absoluteFilePath, placeOf } from "./files"; import { RpcFailure } from "./room"; @@ -29,6 +31,8 @@ const MAX_UPLOADS = 8; const MAX_ACTIVE_CHUNKS = 4; const MAX_RANGES = 4096; const MAX_IDLE_MS = 120_000; +const STALE_TEMP_MS = 24 * 60 * 60_000; +const TEMP_UPLOAD = /^\.nexus-push-[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\.tmp$/; const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/; type Begin = RpcParams<"files.write.begin">; @@ -61,7 +65,9 @@ interface Upload { work: Set>; ranges: Range[]; sealed: boolean; + started: boolean; destination?: Destination; + agent?: { writer: AgentFileWrites; path: string }; temporary?: string; file?: FileHandle; fileInfo?: Stats; @@ -100,18 +106,23 @@ function optionalStat(target: string): Promise { }); } +export function workspaceWritePath(root: string, requested: string, directory: boolean): string { + const place = placeOf(root, [], requested); + if (place.kind !== "workspace") throw forbidden("the path is outside the workspace's folder"); + if (!place.rel && !directory) throw invalid("the destination is a folder"); + return place.rel; +} + async function locate(root: string, request: Scope, directory = false): Promise { let asked: string; if (request.anywhere) { asked = absoluteFilePath(request.path); } else { - const place = placeOf(root, [], request.path); - if (place.kind !== "workspace") throw forbidden("the path is outside the workspace's folder"); - if (!place.rel && !directory) throw invalid("the destination is a folder"); - asked = absoluteFilePath(path.join(root, place.rel)); - if (place.rel) { + const rel = workspaceWritePath(root, request.path, directory); + asked = absoluteFilePath(path.join(root, rel)); + if (rel) { let prefix = root; - for (const part of place.rel.split("/").slice(0, -1)) { + for (const part of rel.split("/").slice(0, -1)) { prefix = path.join(prefix, part); if ((await lstat(prefix)).isSymbolicLink()) { throw forbidden("the path goes through a symbolic link"); @@ -198,6 +209,31 @@ export async function makeRemoteDirectory( } } +/** Best-effort cleanup of abandoned push files from earlier processes. */ +async function removeStaleUploads( + destination: Destination, + checkDestination: () => Promise, +): Promise { + const cutoff = Date.now() - STALE_TEMP_MS; + const target = path.basename(destination.path).toLowerCase(); + try { + const folder = await opendir(destination.parent); + for await (const entry of folder) { + if (entry.name === target || !entry.isFile() || !TEMP_UPLOAD.test(entry.name)) continue; + const temporary = path.join(destination.parent, entry.name); + const info = await optionalStat(temporary).catch(() => undefined); + if (!info?.isFile() || info.mtimeMs >= cutoff) continue; + await checkDestination(); + const current = await optionalStat(temporary).catch(() => undefined); + if (current?.isFile() && current.mtimeMs < cutoff && sameFile(info, current)) { + await unlink(temporary).catch(() => {}); + } + } + } catch { + // Listing access is optional; begin rechecks the destination before it creates a file. + } +} + /** Device-owned uploads, written to disk as pieces arrive and published by rename. */ export class FileWrites { private readonly uploads = new Map(); @@ -209,6 +245,7 @@ export class FileWrites { root: string, request: Begin, authorize: Authorize, + agent?: Upload["agent"], ): Promise> { authorize(); if ( @@ -228,13 +265,29 @@ export class FileWrites { work: new Set(), ranges: [], sealed: false, + started: false, reserved: 0, + agent, }; this.uploads.set(uploadId, upload); this.touch(uploadId, upload); try { await this.work(upload, async () => { + if (agent) { + authorize(agent.path); + await agent.writer.begin(uploadId, agent.path, request.size); + upload.controller.signal.throwIfAborted(); + return; + } const destination = await locate(root, request); + const checkDestination = async () => { + await this.verifyParent(root, request, destination); + upload.controller.signal.throwIfAborted(); + authorize(destination.path); + }; + await existingFile(destination.path); + await checkDestination(); + await removeStaleUploads(destination, checkDestination); const space = await statfs(destination.parent, { bigint: true }); const reserved = [...this.uploads.values()] .filter((other) => other.destination?.parentInfo.dev === destination.parentInfo.dev) @@ -247,10 +300,7 @@ export class FileWrites { } upload.destination = destination; upload.reserved = request.size; - await existingFile(destination.path); - await this.verifyParent(root, request, destination); - upload.controller.signal.throwIfAborted(); - authorize(destination.path); + await checkDestination(); const temporary = path.join(destination.parent, `.nexus-push-${uploadId}.tmp`); upload.file = await open(temporary, "wx+", 0o600); upload.temporary = temporary; @@ -258,6 +308,7 @@ export class FileWrites { await this.verifyParent(root, request, destination); upload.controller.signal.throwIfAborted(); }); + upload.started = true; return { uploadId }; } catch (error) { await this.discard(uploadId, upload); @@ -267,7 +318,7 @@ export class FileWrites { async chunk(device: string, request: RpcParams<"files.write.chunk">): Promise { const upload = this.mine(device, request); - if (upload.sealed || !upload.file) throw invalid("the upload is not accepting pieces"); + if (upload.sealed || !upload.started) throw invalid("the upload is not accepting pieces"); if (upload.work.size >= MAX_ACTIVE_CHUNKS) throw busy(); if (!BASE64.test(request.data)) throw invalid("the piece is not base64"); const bytes = Buffer.from(request.data, "base64"); @@ -281,12 +332,16 @@ export class FileWrites { const end = request.offset + bytes.length; if (end > upload.request.size) throw invalid("the piece exceeds the declared file size"); cover(upload.ranges, request.offset, end); - const file = upload.file; this.touch(request.uploadId, upload); try { await this.work(upload, async () => { - await writeFileChunk(file, bytes, request.offset); - upload.reserved -= bytes.length; + if (upload.agent) { + await upload.agent.writer.chunk(request.uploadId, request.offset, request.data); + } else { + if (!upload.file) throw invalid("the upload has not started"); + await writeFileChunk(upload.file, bytes, request.offset); + upload.reserved -= bytes.length; + } }); } catch (error) { await this.discard(request.uploadId, upload); @@ -296,10 +351,9 @@ export class FileWrites { async end(device: string, request: RpcParams<"files.write.end">): Promise { const upload = this.mine(device, request); - const { file, destination, temporary, fileInfo } = upload; + const { agent } = upload; if (upload.sealed || upload.work.size > 0) throw busy(); - if (!file || !destination || !temporary || !fileInfo) - throw invalid("the upload has not started"); + if (!upload.started) throw invalid("the upload has not started"); upload.sealed = true; upload.cancelExpiry?.(); const signal = upload.controller.signal; @@ -307,38 +361,25 @@ export class FileWrites { return await this.work(upload, async () => { const size = upload.request.size; if ( - (size !== 0 && - (upload.ranges.length !== 1 || - upload.ranges[0]?.[0] !== 0 || - upload.ranges[0][1] !== size)) || - (await file.stat()).size !== size + size !== 0 && + (upload.ranges.length !== 1 || + upload.ranges[0]?.[0] !== 0 || + upload.ranges[0][1] !== size) ) { throw invalid("the upload did not receive all the declared bytes"); } - const hash = createHash("sha256"); - for (let offset = 0; offset < size; offset += MAX_BYTES_CHUNK) { + let destination: string; + if (agent) { + await agent.writer.verify(request.uploadId, request.sha256, size); signal.throwIfAborted(); - hash.update(await readFileChunk(file, offset, Math.min(MAX_BYTES_CHUNK, size - offset))); - } - const sha256 = hash.digest("hex"); - if (sha256 !== request.sha256) throw invalid("the upload's SHA-256 did not match"); - await this.verifyParent(upload.root, upload.request, destination); - const current = await existingFile(destination.path); - if (process.platform !== "win32") { - const mode = current ? current.mode & 0o7777 : 0o666 & ~process.umask(); - await file.chmod(mode); - } - await file.close(); - delete upload.file; - await this.verifyParent(upload.root, upload.request, destination); - if (!sameFile(fileInfo, await lstat(temporary))) { - throw forbidden("the temporary file changed during the upload"); + upload.authorize(agent.path); + await agent.writer.commit(request.uploadId); + delete upload.agent; + destination = agent.path; + } else { + destination = await this.commitLocal(upload, request.sha256); } - signal.throwIfAborted(); - upload.authorize(destination.path); - await rename(temporary, destination.path); - delete upload.temporary; - const result = { path: destination.path, size, sha256 }; + const result = { path: destination, size, sha256: request.sha256 }; log.info( `Received remote file ${JSON.stringify({ device, from: upload.request.from, ...result })}`, ); @@ -351,6 +392,40 @@ export class FileWrites { } } + private async commitLocal(upload: Upload, sha256: string): Promise { + const { file, destination, temporary, fileInfo } = upload; + if (!file || !destination || !temporary || !fileInfo) + throw invalid("the upload has not started"); + const { size } = upload.request; + if ((await file.stat()).size !== size) { + throw invalid("the upload did not receive all the declared bytes"); + } + const signal = upload.controller.signal; + const hash = createHash("sha256"); + for (let offset = 0; offset < size; offset += MAX_BYTES_CHUNK) { + signal.throwIfAborted(); + hash.update(await readFileChunk(file, offset, Math.min(MAX_BYTES_CHUNK, size - offset))); + } + if (hash.digest("hex") !== sha256) throw invalid("the upload's SHA-256 did not match"); + await this.verifyParent(upload.root, upload.request, destination); + const current = await existingFile(destination.path); + if (process.platform !== "win32") { + const mode = current ? current.mode & 0o7777 : 0o666 & ~process.umask(); + await file.chmod(mode); + } + await file.close(); + delete upload.file; + await this.verifyParent(upload.root, upload.request, destination); + if (!sameFile(fileInfo, await lstat(temporary))) { + throw forbidden("the temporary file changed during the upload"); + } + signal.throwIfAborted(); + upload.authorize(destination.path); + await rename(temporary, destination.path); + delete upload.temporary; + return destination.path; + } + async abort(device: string, request: UploadRef): Promise { const upload = this.uploads.get(request.uploadId); if (upload?.device !== device) return; @@ -409,7 +484,16 @@ export class FileWrites { upload.cancelExpiry?.(); upload.cleanup = (async () => { try { - await Promise.allSettled(upload.work); + if (upload.started && upload.agent) { + const [abort] = await Promise.allSettled([ + upload.agent.writer.abort(id), + Promise.allSettled(upload.work), + ]); + if (abort.status === "rejected") throw abort.reason; + } else { + await Promise.allSettled(upload.work); + await upload.agent?.writer.abort(id); + } try { await upload.file?.close(); } finally { diff --git a/src/main/features/remote/serving/files.ts b/src/main/features/remote/serving/files.ts index 205859a88..4357b12c8 100644 --- a/src/main/features/remote/serving/files.ts +++ b/src/main/features/remote/serving/files.ts @@ -6,6 +6,8 @@ import { type RpcParams, } from "../../../../../packages/remote-protocol/src"; import type { CliServices } from "../../cli/context"; +import { isAgentBackedProvider } from "../../fs/bridge/provider"; +import { AgentFileWrites } from "../agent-file-writes"; import type { AgentGate } from "../agent-gate"; import type { ConsentControls } from "../consents"; import { type FileWrites, fileWriteDestination, makeRemoteDirectory } from "../file-writes"; @@ -65,16 +67,28 @@ export async function serveFileDestination( if (!current || current.rootPath !== root || !isDeepStrictEqual(current.location, location)) { throw new RpcFailure(RpcErrorCode.NotFound, "no such workspace"); } - if (location.kind !== "local") { + if (write.anywhere && location.kind !== "local") { throw new RpcFailure( RpcErrorCode.Unavailable, - "file writes require a local workspace on the serving computer", + "absolute file access requires a local workspace on the serving computer", ); } }; authorize(); const directory = method === "files.mkdir"; - approvedPath = await fileWriteDestination(root, write, directory); + let agent: AgentFileWrites | undefined; + if (location.kind !== "local") { + const fs = await watches.files?.fs(write.workspaceId).catch(unavailable); + if (!fs || fs.kind !== location.kind || !isAgentBackedProvider(fs)) { + throw new RpcFailure( + RpcErrorCode.Unavailable, + "the workspace's filesystem agent is unavailable", + ); + } + agent = new AgentFileWrites(fs, root, write.path, directory); + } + approvedPath = await (agent?.resolve() ?? fileWriteDestination(root, write, directory)); + authorize(); await watches.agents?.checkFile( s, { @@ -89,8 +103,16 @@ export async function serveFileDestination( }, ); authorize(); - if (directory) return makeRemoteDirectory(root, write, authorize); - return fileWrites.begin(from, root, write, authorize); + if (directory) { + return agent ? agent.mkdir(approvedPath) : makeRemoteDirectory(root, write, authorize); + } + return fileWrites.begin( + from, + root, + write, + authorize, + agent && { writer: agent, path: approvedPath }, + ); } /** Lists or reads a file with the request's workspace and access checks. */ diff --git a/tests/unit/main/features/remote/file-writes.test.ts b/tests/unit/main/features/remote/file-writes.test.ts index 6edfdbad0..2e95edf02 100644 --- a/tests/unit/main/features/remote/file-writes.test.ts +++ b/tests/unit/main/features/remote/file-writes.test.ts @@ -10,6 +10,7 @@ import { rm, stat, symlink, + utimes, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -20,7 +21,10 @@ import { type RpcParams, } from "../../../../../packages/remote-protocol/src"; import type { CliServices } from "../../../../../src/main/features/cli/context"; +import { AgentFsProvider } from "../../../../../src/main/features/fs/bridge/agent-provider"; +import type { FsProvider } from "../../../../../src/main/features/fs/bridge/provider"; import { AgentGate } from "../../../../../src/main/features/remote/agent-gate"; +import { fileVerifyTimeoutMs } from "../../../../../src/main/features/remote/file-io"; import { FileWrites, makeRemoteDirectory, @@ -207,6 +211,23 @@ test("another peer cannot complete or abort an upload; owner abort is idempotent expect(await readdir(root)).toEqual([]); }); +test.each([ + ["abandoned file", ".nexus-push-a0000000-0000-4000-8000-000000000000.tmp", "old", false], + ["recent file", ".nexus-push-b0000000-0000-4000-8000-000000000000.tmp", "recent", true], + ["unrelated file", ".nexus-push-not-an-upload.tmp", "old", true], + ["matching directory", ".nexus-push-c0000000-0000-4000-8000-000000000000.tmp", "dir", true], +] as const)("begin handles %s in the destination folder", async (_, name, kind, kept) => { + const target = path.join(root, name); + if (kind === "dir") await mkdir(target); + else await writeFile(target, ""); + if (kind !== "recent") { + const old = new Date(Date.now() - 25 * 60 * 60_000); + await utimes(target, old, old); + } + await begin("data", 0); + expect((await readdir(root)).includes(name)).toBe(kept); +}); + test("idle expiry is reset by a chunk and removes the temporary file", async () => { const pending = new Set<() => Promise>(); writes = new FileWrites((expire, idleMs) => { @@ -245,11 +266,28 @@ test("reserves concurrent slots before checking files and releases them when upl function consentHost( policy: RemoteAgentsPolicy, ...answers: Array<"once" | "session" | "deny" | "timeout"> +) { + return fileHost(policy, answers); +} + +function fileHost( + policy: RemoteAgentsPolicy, + answers: Array<"once" | "session" | "deny" | "timeout">, + agent?: FsProvider, ) { const app = fakeApp(); const workspace = app.workspaces[0]; if (!workspace) throw new Error("workspace fixture is missing"); - Object.assign(workspace, { rootPath: root, location: { kind: "local", rootPath: root } }); + const location = + agent?.kind === "ssh" + ? { kind: "ssh", host: "host", remotePath: "/remote" } + : agent?.kind === "wsl" + ? { kind: "wsl", distro: "Ubuntu", remotePath: "/remote" } + : { kind: "local", rootPath: root }; + Object.assign(workspace, { + rootPath: "remotePath" in location ? location.remotePath : root, + location, + }); const prompts: unknown[] = []; const services = { ...app.services, @@ -266,6 +304,7 @@ function consentHost( fileWrites: writes, files: { fs: async () => { + if (agent) return agent; throw new Error("probing a write target reads no entries"); }, shrink: () => null, @@ -344,3 +383,65 @@ test("recursive push outside the workspace can approve the destination probe and change: { kind: "fileWriteTree", detail: dest, terminal: true }, }); }); + +test.each([ + ["ssh", true], + ["wsl", true], + ["ssh", false], + ["wsl", false], +] as const)("%s workspace push with upload RPC support %s", async (kind, supported) => { + const fs = new AgentFsProvider(kind); + const calls: Array<{ method: string; params: unknown; timeoutMs?: number }> = []; + fs.callAgentMethod = async ( + method: string, + params?: unknown, + options?: { timeoutMs?: number }, + ): Promise => { + calls.push({ method, params, ...options }); + if (!supported) + throw Object.assign(new Error("method unavailable"), { code: "unsupported-method" }); + if (method === "fs.upload.resolve") { + return { path: `/remote/${(params as { relPath: string }).relPath}` } as T; + } + return {} as T; + }; + const { invoke, prompts } = fileHost("ask", ["once"], fs); + if (!supported) { + for (const method of ["files.write.begin", "files.mkdir"] as const) { + await expect(invoke(method, { path: "folder", size: 3 })).rejects.toMatchObject({ + code: RpcErrorCode.Unavailable, + message: "the remote filesystem agent must be updated to write files", + }); + } + expect(prompts).toHaveLength(0); + expect(calls.every(({ method }) => method === "fs.upload.resolve")).toBe(true); + return; + } + if (kind === "ssh") expect(fileVerifyTimeoutMs(8 * 1024 ** 3)).toBe(286_000); + expect(await invoke("files.mkdir", { path: "folder" })).toEqual({ path: "/remote/folder" }); + const { uploadId } = (await invoke("files.write.begin", { + path: "/remote/folder/data", + size: 3, + })) as { uploadId: string }; + const chunk = piece(uploadId, 0, "new"); + await invoke("files.write.chunk", chunk); + const sha256 = digest("new"); + expect(await invoke("files.write.end", { uploadId, sha256 })).toEqual({ + path: "/remote/folder/data", + size: 3, + sha256, + }); + expect(prompts).toHaveLength(1); + expect(calls).toEqual([ + { method: "fs.upload.resolve", params: { relPath: "folder", directory: true } }, + { method: "fs.upload.mkdir", params: { relPath: "folder", expectedPath: "/remote/folder" } }, + { method: "fs.upload.resolve", params: { relPath: "folder/data", directory: false } }, + { + method: "fs.upload.begin", + params: { relPath: "folder/data", expectedPath: "/remote/folder/data", uploadId, size: 3 }, + }, + { method: "fs.upload.chunk", params: chunk }, + { method: "fs.upload.verify", params: { uploadId, sha256 }, timeoutMs: 30_001 }, + { method: "fs.upload.commit", params: { uploadId } }, + ]); +});