diff --git a/.changeset/dofs-readdir-entries.md b/.changeset/dofs-readdir-entries.md new file mode 100644 index 00000000..83611a05 --- /dev/null +++ b/.changeset/dofs-readdir-entries.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/dofs": patch +--- + +`SQLiteWorkspaceProvider.readdirEntriesSync(path)` lists a directory with each entry's type, permission bits and inode, as an `lstat` of it reports them, from one query. diff --git a/.changeset/readdir-entry-types.md b/.changeset/readdir-entry-types.md new file mode 100644 index 00000000..330c1fe1 --- /dev/null +++ b/.changeset/readdir-entry-types.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computerd": patch +--- + +Directory listings on the mount now report each entry's type and inode, on synced and local-only paths alike. Before, every entry came back as `DT_UNKNOWN`, which `bun install` treats as nothing to link: with its cache under a local-only path such as `/workspace/.cache`, it left packages out of `node_modules` and failed with `failed to enqueue lifecycle scripts ...: ENOENT`. Tools that read `d_type`, such as `find` and `git`, no longer need an `lstat` per entry to learn it. diff --git a/packages/computerd/src/fuse/driver.test.ts b/packages/computerd/src/fuse/driver.test.ts index 67c5e737..79b4637e 100644 --- a/packages/computerd/src/fuse/driver.test.ts +++ b/packages/computerd/src/fuse/driver.test.ts @@ -1069,3 +1069,69 @@ test("FUSE ftruncate does not depend on fuse-native binding this", async () => { expect(await status((cb) => ops.flush("/ftruncate-this.txt", fh, cb))).toBe(0); expect(Buffer.from(vfs.readFileSync("/ftruncate-this.txt")).toString("utf8")).toBe("abc"); }); + +for (const listing of ["batched", "per entry"] as const) { + test(`readdir gives each entry the type and inode getattr reports (${listing})`, async () => { + // libfuse takes d_type from these stats. Without them every entry is + // DT_UNKNOWN, and readers that trust d_type skip it: bun's hardlink + // install walking a cache on the mount linked nothing. + const { vfs } = await createNodeVirtualFileSystem(); + vfs.mkdirSync("/dir/sub", { recursive: true }); + vfs.writeFileSync("/dir/file.txt", Buffer.from("x")); + vfs.symlinkSync("file.txt", "/dir/link"); + // "per entry" is a VFS without the batched listing, which the driver + // stats name by name. + const source = + listing === "batched" + ? vfs + : new Proxy(vfs, { + // Bound to the real object: the VFS keeps private fields, + // which a call with the proxy as `this` cannot reach. + get: (target, property) => { + if (property === "readdirEntriesSync") return undefined; + const value: unknown = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const ops = makeFUSEOps(source); + expect(await status((cb) => ops.link("/dir/file.txt", "/dir/hard.txt", cb))).toBe(0); + expect(await status((cb) => ops.chmod("/dir/file.txt", 0o700, cb))).toBe(0); + // Created through the driver and not yet flushed, so only the driver + // knows about it. + expect((await callback((cb) => ops.create("/dir/pending.txt", 0o644, cb))).errno).toBe(0); + + const result = await new Promise<{ errno: number; names: string[]; stats: unknown[] }>( + (resolve) => + ops.readdir("/dir", (errno, names, stats = []) => resolve({ errno, names, stats })), + ); + expect(result.errno).toBe(0); + expect([...result.names].sort()).toEqual([ + "file.txt", + "hard.txt", + "link", + "pending.txt", + "sub", + ]); + expect(result.stats).toHaveLength(result.names.length); + + const S_IFMT = 0o170000; + const types: Record = {}; + for (const [i, name] of result.names.entries()) { + const fromListing = result.stats[i] as { mode: number; ino: number }; + const fromGetattr = (await callback((cb) => ops.getattr(`/dir/${name}`, cb))).result as { + mode: number; + ino: number; + }; + expect(fromListing.mode, name).toBe(fromGetattr.mode); + expect(fromListing.ino, name).toBe(fromGetattr.ino); + types[name] = fromListing.mode & S_IFMT; + } + expect(types).toEqual({ + "file.txt": 0o100000, + "hard.txt": 0o100000, + link: 0o120000, + "pending.txt": 0o100000, + sub: 0o040000, + }); + }); +} diff --git a/packages/computerd/src/fuse/driver.ts b/packages/computerd/src/fuse/driver.ts index 1af773cb..3795d519 100644 --- a/packages/computerd/src/fuse/driver.ts +++ b/packages/computerd/src/fuse/driver.ts @@ -1,5 +1,7 @@ import { writeFileSync as nodeWriteFileSync } from "node:fs"; import { posix } from "node:path"; + +import type { WorkspaceDirentEntry } from "@cloudflare/dofs"; import type { FUSEBackend } from "./backend.js"; import { buildFuseOptionString } from "./options.js"; import { @@ -41,6 +43,12 @@ const STAT_BLOCK_SIZE = 512; // Match what statfs advertises (bsize: 4096) and what the backing VFS // reports so a fabricated stat stays consistent with a persisted one. const PREFERRED_IO_BLOCK_SIZE = 4096; +const DIRENT_TYPE_BITS: Record = { + file: 0o100000, + dir: 0o040000, + symlink: 0o120000, +}; +const EPOCH = new Date(0); function blocksForSize(size: number): number { return size <= 0 ? 0 : Math.ceil(size / STAT_BLOCK_SIZE); @@ -48,12 +56,18 @@ function blocksForSize(size: number): number { type StatusCallback = (errnoOrBytes: number) => void; type ResultCallback = (errno: number, result: T) => void; +/** + * `stats`, when given, is one per name. libfuse takes each entry's type + * (d_type) and, under use_ino, its inode number from it; without it every + * entry is DT_UNKNOWN. + */ +export type ReaddirCallback = (errno: number, names: string[], stats?: FuseStat[]) => void; type NotImplementedOperation = (...args: unknown[]) => void; export interface FuseOps { init(cb?: StatusCallback): void; error: NotImplementedOperation; - readdir(path: string, cb: ResultCallback): void; + readdir(path: string, cb: ReaddirCallback): void; getattr(path: string, cb: ResultCallback): void; fgetattr(path: string, fh: number, cb: ResultCallback): void; open(path: string, flags: number, cb: ResultCallback): void; @@ -285,6 +299,9 @@ export function makeFUSEOps(vfs: NodeVirtualFileSystem, mountPoint = "/"): FuseO const linkableVfs = vfs as NodeVirtualFileSystem & { linkSync?: (existingPath: string, newPath: string) => void; }; + const listingVfs = vfs as NodeVirtualFileSystem & { + readdirEntriesSync?: (path: string) => WorkspaceDirentEntry[]; + }; const markDirty = (entry: FileEntry, start: number, end: number): void => { entry.dirty = true; if (start < end) entry.dirtyRanges.push({ start, end }); @@ -294,6 +311,7 @@ export function makeFUSEOps(vfs: NodeVirtualFileSystem, mountPoint = "/"): FuseO return parent === "." ? "/" : parent; }; const baseName = (path: string): string => posix.basename(path); + const childPath = (parent: string, name: string): string => posix.join(parent, name); const isPendingCreate = (path: string): boolean => files.get(path)?.pendingCreate === true; const exists = (path: string): boolean => isPendingCreate(path) || vfs.existsSync(toVfs(path)); const getBufferStats = (): FuseBufferStats => { @@ -460,6 +478,54 @@ export function makeFUSEOps(vfs: NodeVirtualFileSystem, mountPoint = "/"): FuseO }; }; + function statPath(path: string): FuseStat { + const entry = files.get(path); + const stat = + entry?.pendingCreate === true ? pendingStat(entry) : statNode(vfs.lstatSync(toVfs(path))); + // File content lives outside the VFS, so prefer our size. The + // buffered size can outrun the persisted inode before a spill, + // so recompute block accounting from it to keep st_blocks + // coherent with the size the kernel sees. + if (entry !== undefined) { + stat.size = entry.size; + stat.blocks = blocksForSize(entry.size); + } + const override = meta.get(path); + if (override) { + if (override.mode !== undefined) { + stat.mode = (stat.mode & 0o170000) | (override.mode & 0o7777); + } + if (override.uid !== undefined) stat.uid = override.uid; + if (override.gid !== undefined) stat.gid = override.gid; + if (override.atime) stat.atime = override.atime; + if (override.mtime) stat.mtime = override.mtime; + } + return stat; + } + + // A listing entry's stat. libfuse puts only the type and the inode of a + // plain readdir entry into getdents(2) and caches none of it as + // attributes, so the remaining fields are placeholders. With readdirplus + // these would become cached attributes and need statPath's real values. + function direntStat(path: string, entry: WorkspaceDirentEntry): FuseStat { + let mode = DIRENT_TYPE_BITS[entry.type] | entry.mode; + const override = meta.get(path); + if (override?.mode !== undefined) mode = (mode & 0o170000) | (override.mode & 0o7777); + return { + mtime: EPOCH, + atime: EPOCH, + ctime: EPOCH, + size: 0, + mode, + uid: typeof process.getuid === "function" ? process.getuid() : 0, + gid: typeof process.getgid === "function" ? process.getgid() : 0, + nlink: 1, + ino: entry.inode, + blksize: PREFERRED_IO_BLOCK_SIZE, + blocks: 0, + }; + } + return { init(cb) { cb?.(0); @@ -470,13 +536,34 @@ export function makeFUSEOps(vfs: NodeVirtualFileSystem, mountPoint = "/"): FuseO error: notImplemented("error"), readdir(path, cb) { + // Without a stat per entry, libfuse reports every entry as + // DT_UNKNOWN. POSIX allows that, but readers that trust d_type + // skip such entries: bun's hardlink install walks its cache that + // way and links nothing into node_modules. try { - const entries = new Set(vfs.readdirSync(toVfs(path))); + const stats = new Map(); + const listed = listingVfs.readdirEntriesSync?.(toVfs(path)); + if (listed !== undefined) { + for (const entry of listed) { + stats.set(entry.name, direntStat(childPath(path, entry.name), entry)); + } + } else { + for (const name of vfs.readdirSync(toVfs(path)) as string[]) { + try { + stats.set(name, statPath(childPath(path, name))); + } catch { + // Listed but gone by the time it was stat'd. + } + } + } + // Created through this driver and not yet in the VFS, or staged + // over a committed name; either way getattr answers from here. for (const [pendingPath, entry] of files) { - if (entry.pendingCreate && parentPath(pendingPath) === path) - entries.add(baseName(pendingPath)); + if (entry.pendingCreate && parentPath(pendingPath) === path) { + stats.set(baseName(pendingPath), statPath(pendingPath)); + } } - cb(0, [...entries]); + cb(0, [...stats.keys()], [...stats.values()]); } catch (error) { cb(toErrno(error), []); } @@ -484,28 +571,7 @@ export function makeFUSEOps(vfs: NodeVirtualFileSystem, mountPoint = "/"): FuseO getattr(path, cb) { try { - const entry = files.get(path); - const stat = - entry?.pendingCreate === true ? pendingStat(entry) : statNode(vfs.lstatSync(toVfs(path))); - // File content lives outside the VFS, so prefer our size. The - // buffered size can outrun the persisted inode before a spill, - // so recompute block accounting from it to keep st_blocks - // coherent with the size the kernel sees. - if (entry !== undefined) { - stat.size = entry.size; - stat.blocks = blocksForSize(entry.size); - } - const override = meta.get(path); - if (override) { - if (override.mode !== undefined) { - stat.mode = (stat.mode & 0o170000) | (override.mode & 0o7777); - } - if (override.uid !== undefined) stat.uid = override.uid; - if (override.gid !== undefined) stat.gid = override.gid; - if (override.atime) stat.atime = override.atime; - if (override.mtime) stat.mtime = override.mtime; - } - cb(0, stat); + cb(0, statPath(path)); } catch (error) { cb(toErrno(error), null); } diff --git a/packages/computerd/src/fuse/passthrough.test.ts b/packages/computerd/src/fuse/passthrough.test.ts index 212f55d3..de7a662b 100644 --- a/packages/computerd/src/fuse/passthrough.test.ts +++ b/packages/computerd/src/fuse/passthrough.test.ts @@ -430,6 +430,107 @@ describe("withLocalPassthrough: directory listing", () => { }); }); +describe("withLocalPassthrough: readdir entry types", () => { + // libfuse takes d_type, and under use_ino the inode, from the stat given + // with each name. Without one every entry is DT_UNKNOWN, which readers + // that trust d_type skip: bun's hardlink install walking a cache under a + // local-only path linked nothing. + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "computerd-passthrough-")); + }); + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + const S_IFMT = 0o170000; + type EntryStat = { mode: number; ino: number }; + // The listing as name -> stat, failing unless every name has one. + const list = (ops: FuseOps, path: string) => { + let out = { code: -1, names: [] as string[], byName: new Map() }; + ops.readdir(path, (code, names, stats = []) => { + expect(stats).toHaveLength(names.length); + out = { code, names, byName: new Map(names.map((name, i) => [name, stats[i] as EntryStat])) }; + }); + return out; + }; + const typeOf = (stat: EntryStat | undefined): number | undefined => + stat === undefined ? undefined : stat.mode & S_IFMT; + + test("a local directory reports each entry's lstat type and inode", () => { + const { ops } = withLocalPassthrough(recordingOps().ops, { + root, + ignore: resolveMountIgnore(["**/.cache"], MOUNT), + mountPoint: MOUNT, + }); + const dir = join(root, ".cache/bun/react@19"); + mkdirSync(join(dir, "cjs"), { recursive: true }); + writeFileSync(join(dir, "package.json"), "{}"); + symlinkSync("package.json", join(dir, "link.json")); + + const { code, names, byName } = list(ops, "/.cache/bun/react@19"); + expect(code).toBe(0); + expect(typeOf(byName.get("cjs"))).toBe(constants.S_IFDIR); + expect(typeOf(byName.get("package.json"))).toBe(constants.S_IFREG); + expect(typeOf(byName.get("link.json"))).toBe(constants.S_IFLNK); + for (const name of names) { + expect(byName.get(name)?.ino, name).toBe(lstatSync(join(dir, name)).ino); + } + }); + + test("a synced directory keeps the VFS stats and adds local children's", () => { + const vfsStat = { mode: constants.S_IFREG | 0o644, ino: 42 }; + const source = recordingOps(); + const vfsOps = new Proxy(source.ops, { + get(target, property: string) { + if (property === "readdir") { + return (_path: string, cb: (code: number, names: string[], stats?: unknown[]) => void) => + cb(0, ["src.ts"], [vfsStat]); + } + return Reflect.get(target, property); + }, + }); + const { ops } = withLocalPassthrough(vfsOps, { + root, + ignore: resolveMountIgnore(["**/node_modules"], MOUNT), + mountPoint: MOUNT, + }); + mkdirSync(join(root, "app/node_modules"), { recursive: true }); + + const { code, byName } = list(ops, "/app"); + expect(code).toBe(0); + expect(byName.get("src.ts")).toBe(vfsStat); + expect(typeOf(byName.get("node_modules"))).toBe(constants.S_IFDIR); + expect(byName.get("node_modules")?.ino).toBe(lstatSync(join(root, "app/node_modules")).ino); + }); + + test("a local-only child shadows a synced entry of the same name", () => { + // getattr routes the name to the local side, so the listing has to + // agree with it, including when stale synced content remains. + const source = recordingOps(); + const vfsOps = new Proxy(source.ops, { + get(target, property: string) { + if (property === "readdir") { + return (_path: string, cb: (code: number, names: string[], stats?: unknown[]) => void) => + cb(0, ["node_modules"], [{ mode: constants.S_IFREG | 0o644, ino: 7 }]); + } + return Reflect.get(target, property); + }, + }); + const { ops } = withLocalPassthrough(vfsOps, { + root, + ignore: resolveMountIgnore(["**/node_modules"], MOUNT), + mountPoint: MOUNT, + }); + mkdirSync(join(root, "node_modules"), { recursive: true }); + + const { names, byName } = list(ops, "/"); + expect(names).toEqual(["node_modules"]); + expect(typeOf(byName.get("node_modules"))).toBe(constants.S_IFDIR); + }); +}); + describe("withLocalPassthrough: symlinks", () => { let root: string; diff --git a/packages/computerd/src/fuse/passthrough.ts b/packages/computerd/src/fuse/passthrough.ts index 3177a4d2..c0ab7c41 100644 --- a/packages/computerd/src/fuse/passthrough.ts +++ b/packages/computerd/src/fuse/passthrough.ts @@ -232,24 +232,44 @@ export function withLocalPassthrough( // A VFS directory may still contain local-only children: the // entries live on disk but the parent does not. Merge both // sides so `ls` shows what a command inside the container sees. - ops.readdir(path, (code, names) => { + ops.readdir(path, (code, names, stats) => { if (code !== 0) { - cb(code, names); + cb(code, names, stats); return; } const extra = localChildren(path); if (extra.length === 0) { - cb(0, names); + cb(0, names, stats); return; } - const merged = new Set([...(names ?? []), ...extra]); - cb(0, [...merged]); + if (stats === undefined || stats.length !== names.length) { + const merged = new Set([...(names ?? []), ...extra]); + cb(0, [...merged]); + return; + } + // A local-only child shadows a synced entry of the same name, + // as getattr does, so its stat is the one reported. + const merged = new Map(); + names.forEach((name, i) => { + const stat = stats[i]; + if (stat !== undefined) merged.set(name, stat); + }); + for (const [name, stat] of statLocalEntries(localPath(path), extra)) { + merged.set(name, stat); + } + cb(0, [...merged.keys()], [...merged.values()]); }); return; } localOps += 1; try { - cb(0, fs.readdirSync(localPath(path))); + const directory = localPath(path); + const entries = statLocalEntries(directory, fs.readdirSync(directory) as string[]); + cb( + 0, + entries.map(([name]) => name), + entries.map(([, stat]) => stat), + ); } catch (error) { cb(toErrno(error), []); } @@ -758,6 +778,21 @@ export function withLocalPassthrough( return names.filter((name) => isLocal(parent === "" ? name : `${parent}/${name}`)); } + // lstat, as getattr does, so the type and inode match what a lookup of + // the same name returns. An entry removed between the listing and the + // stat is left out. + function statLocalEntries(directory: string, names: string[]): Array<[string, FuseStat]> { + const entries: Array<[string, FuseStat]> = []; + for (const name of names) { + try { + entries.push([name, statToFuse(fs.lstatSync(join(directory, name)))]); + } catch { + // Gone since the listing. + } + } + return entries; + } + function hasLocalContents(path: string): boolean { try { return fs.readdirSync(localPath(path)).length > 0; diff --git a/packages/computerd/src/fuse/vfs.ts b/packages/computerd/src/fuse/vfs.ts index 753f2492..d94958e6 100644 --- a/packages/computerd/src/fuse/vfs.ts +++ b/packages/computerd/src/fuse/vfs.ts @@ -54,6 +54,7 @@ const EXTRA_VFS_METHODS = [ "openWriteBufferSync", "openWriteBufferForCreateSync", "releaseWriteBufferSync", + "readdirEntriesSync", ] as const; export interface NodeVFSHandle { diff --git a/packages/dofs/src/fs/readdir.ts b/packages/dofs/src/fs/readdir.ts index 61df4c67..8d2a51e3 100644 --- a/packages/dofs/src/fs/readdir.ts +++ b/packages/dofs/src/fs/readdir.ts @@ -23,6 +23,26 @@ interface DirentRow { link_target: string | null; } +/** + * One directory entry with what an `lstat` of it reports for type, + * permission bits and inode, read with the listing rather than per entry. + */ +export interface WorkspaceDirentEntry { + name: string; + type: "file" | "dir" | "symlink"; + /** Permission bits only (`& 0o7777`). */ + mode: number; + /** 0 for a file still in a pending-create buffer, as `lstat` reports it. */ + inode: number; +} + +interface EntryRow { + inode: number; + name: string; + type: "file" | "dir" | "symlink"; + mode: number; +} + export interface ReaddirOptions { /** Maximum entries to return. */ limit?: number; @@ -174,3 +194,50 @@ function compareByName(a: WorkspaceDirentResult, b: WorkspaceDirentResult): numb } return left.byteLength - right.byteLength; } + +/** + * List a directory with each entry's type, permission bits and inode in one + * query. `readdir` leaves the inode out and costs a write-buffer lookup per + * file for its size; a caller that needs an `lstat` per entry for the type + * would otherwise resolve and stat every child. + * + * A pending-create buffer shadows a committed entry of the same name, as it + * does for `lstat`. + */ +export function readdirEntries(db: Database, path: string): WorkspaceDirentEntry[] { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + + const rows = db.all( + `SELECT n.inode AS inode, + d.name AS name, + n.type AS type, + n.mode AS mode + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, + node.inode, + ); + const entries = new Map(); + for (const row of rows) { + entries.set(row.name, { + name: row.name, + type: row.type, + mode: row.mode & 0o7777, + inode: row.inode, + }); + } + for (const entry of listPendingByParent(db, node.inode)) { + const name = entry.pending?.leafName; + if (name === undefined) continue; + entries.set(name, { name, type: "file", mode: entry.mode & 0o7777, inode: 0 }); + } + return [...entries.values()]; +} diff --git a/packages/dofs/src/index.ts b/packages/dofs/src/index.ts index 20ddb1ba..cc3bd5ed 100644 --- a/packages/dofs/src/index.ts +++ b/packages/dofs/src/index.ts @@ -22,7 +22,7 @@ export { invalidateReadOnlyMountCache, readOnlyRootFor, } from "./fs/mount-guard.js"; -export type { ReaddirOptions, WorkspaceDirentResult } from "./fs/readdir.js"; +export type { ReaddirOptions, WorkspaceDirentEntry, WorkspaceDirentResult } from "./fs/readdir.js"; export type { ReadFileOptions } from "./fs/readFile.js"; export { readlink } from "./fs/readlink.js"; export { rename } from "./fs/rename.js"; diff --git a/packages/dofs/src/provider.test.ts b/packages/dofs/src/provider.test.ts index a14a53f6..2103d131 100644 --- a/packages/dofs/src/provider.test.ts +++ b/packages/dofs/src/provider.test.ts @@ -981,3 +981,41 @@ describe("SQLiteWorkspaceProvider — cached vfs_nodes.size", () => { }); }); }); + +describe("SQLiteWorkspaceProvider — readdirEntriesSync", () => { + it("reports the type, permission bits and inode lstat gives each entry", async () => { + await withProvider((p) => { + p.mkdirSync("/dir/sub", { recursive: true }); + p.writeFileSync("/dir/file.txt", "x"); + p.chmodSync("/dir/file.txt", 0o755); + p.symlinkSync("file.txt", "/dir/link"); + p.linkSync("/dir/file.txt", "/dir/hard.txt"); + p.openWriteBufferForCreateSync("/dir/pending.txt", { mode: 0o600 }); + + const entries = p.readdirEntriesSync("/dir"); + expect(entries.map((entry) => entry.name).sort()).toEqual([ + "file.txt", + "hard.txt", + "link", + "pending.txt", + "sub", + ]); + const typeOf = { file: 0o100000, dir: 0o040000, symlink: 0o120000 }; + for (const entry of entries) { + const stat = p.lstatSync(`/dir/${entry.name}`); + expect(typeOf[entry.type] | entry.mode, entry.name).toBe(stat.mode); + expect(entry.inode, entry.name).toBe(stat.ino); + } + const byName = new Map(entries.map((entry) => [entry.name, entry])); + expect(byName.get("hard.txt")?.inode).toBe(byName.get("file.txt")?.inode); + }); + }); + + it("refuses a missing path and a file", async () => { + await withProvider((p) => { + p.writeFileSync("/file", "x"); + expect(() => p.readdirEntriesSync("/missing")).toThrow(/ENOENT|no such path/); + expect(() => p.readdirEntriesSync("/file")).toThrow(/ENOTDIR|not a directory/); + }); + }); +}); diff --git a/packages/dofs/src/provider.ts b/packages/dofs/src/provider.ts index 1306f834..f864e952 100644 --- a/packages/dofs/src/provider.ts +++ b/packages/dofs/src/provider.ts @@ -13,7 +13,7 @@ import { link as linkImpl } from "./fs/link.js"; import type { MkdirOptions } from "./fs/mkdir.js"; import { mkdir as mkdirImpl } from "./fs/mkdir.js"; import { findPendingWriteBuffer } from "./fs/pendingWriteBuffer.js"; -import { readdir as readdirImpl } from "./fs/readdir.js"; +import { readdir as readdirImpl, readdirEntries, type WorkspaceDirentEntry } from "./fs/readdir.js"; import { readRangeSync as readRangeSyncImpl } from "./fs/readFile.js"; import { readlink as readlinkImpl } from "./fs/readlink.js"; import { rename as renameImpl } from "./fs/rename.js"; @@ -245,6 +245,15 @@ export class SQLiteWorkspaceProvider { return entries.map((entry) => entry.name); } + /** + * The directory's entries with the type, permission bits and inode an + * `lstat` of each reports, from one query. Not part of the + * `@platformatic/vfs` provider surface. + */ + readdirEntriesSync(path: string): WorkspaceDirentEntry[] { + return readdirEntries(this.db, path); + } + mkdir(path: string, options?: MkdirOptions): Promise { return Promise.resolve(this.mkdirSync(path, options)); } diff --git a/script/fs-tests.sh b/script/fs-tests.sh index a5097bc1..ca457a57 100755 --- a/script/fs-tests.sh +++ b/script/fs-tests.sh @@ -32,6 +32,47 @@ else fi export STAT_SIZE STAT_MTIME STAT_ATIME STAT_NLINK STAT_INO STAT_MODE +# Print "name=type" for each entry of a directory, using the d_type that +# readdir(3) returns. Node cannot show this: it lstats an entry whose type +# is DT_UNKNOWN. bun does not, and skips such entries. +dirent_types() { + python3 - "$1" <<'PY' +import ctypes, sys + +class Dirent(ctypes.Structure): + # struct dirent on 64-bit glibc. + _fields_ = [ + ("d_ino", ctypes.c_uint64), + ("d_off", ctypes.c_int64), + ("d_reclen", ctypes.c_ushort), + ("d_type", ctypes.c_ubyte), + ("d_name", ctypes.c_char * 256), + ] + +libc = ctypes.CDLL(None) +libc.opendir.restype = ctypes.c_void_p +libc.opendir.argtypes = [ctypes.c_char_p] +libc.readdir.restype = ctypes.POINTER(Dirent) +libc.readdir.argtypes = [ctypes.c_void_p] +libc.closedir.argtypes = [ctypes.c_void_p] + +handle = libc.opendir(sys.argv[1].encode()) +if not handle: + sys.exit(f"opendir {sys.argv[1]} failed") +kinds = {4: "dir", 8: "file", 10: "symlink"} +entries = [] +while True: + entry = libc.readdir(handle) + if not entry: + break + name = entry.contents.d_name.decode() + if name not in (".", ".."): + entries.append(f"{name}={kinds.get(entry.contents.d_type, 'unknown')}") +libc.closedir(handle) +print(" ".join(sorted(entries))) +PY +} + run() { # run NAME -- COMMANDS... # Each test executes in its own subdir under $ROOT with set -e. @@ -218,6 +259,21 @@ run "readdir returns sorted entries" -- ' listing=$(ls ld | sort | tr "\n" " ") [ "$listing" = "a b c " ] ' +if command -v python3 >/dev/null 2>&1; then + run "readdir reports entry types" -- ' + mkdir -p ld/sub && touch ld/file && ln -s file ld/link + [ "$(dirent_types ld)" = "file=file link=symlink sub=dir" ] + ' + # Local-only when computerd runs with MOUNT_IGNORE=**/node_modules, as + # run-fs-tests.sh starts it; synced otherwise. + run "readdir reports entry types under node_modules" -- ' + mkdir -p node_modules/pkg/sub && touch node_modules/pkg/file + ln -s file node_modules/pkg/link + [ "$(dirent_types node_modules/pkg)" = "file=file link=symlink sub=dir" ] + ' +else + skip_test "readdir reports entry types" "python3 not installed" +fi run "find -type f" -- ' mkdir ld && touch ld/a ld/b ld/c [ "$(find ld -type f | wc -l)" = "3" ] diff --git a/script/run-fs-tests.sh b/script/run-fs-tests.sh index ef397160..4da14ea3 100755 --- a/script/run-fs-tests.sh +++ b/script/run-fs-tests.sh @@ -6,10 +6,13 @@ # fs-tests' status code. set -u apt-get update >/dev/null 2>&1 -apt-get install -y --no-install-recommends fuse3 libfuse2t64 attr util-linux coreutils findutils git ca-certificates curl >/dev/null 2>&1 +apt-get install -y --no-install-recommends fuse3 libfuse2t64 attr util-linux coreutils findutils git ca-certificates curl python3 >/dev/null 2>&1 mkdir -p /tmp/workspace -PORT=45678 MOUNT_POINT=/tmp/workspace /usr/local/bin/computerd >/tmp/computerd.log 2>&1 & +# node_modules is local-only, so fs-tests covers that layer as well as the +# synced one. +PORT=45678 MOUNT_POINT=/tmp/workspace MOUNT_IGNORE='**/node_modules' \ + /usr/local/bin/computerd >/tmp/computerd.log 2>&1 & COMPUTERD_PID=$! # Wait for /health diff --git a/script/shell b/script/shell index b514ece0..c4ca4cf9 100755 --- a/script/shell +++ b/script/shell @@ -23,4 +23,4 @@ exec docker run --rm -it \ -v "${repo_root}/script/fs-bench.sh:/usr/local/bin/fs-bench:ro" \ -w /root \ debian:stable-slim \ - bash -c 'apt-get update >/dev/null 2>&1 && apt-get install -y --no-install-recommends fuse3 libfuse2t64 attr util-linux coreutils findutils git ca-certificates >/dev/null 2>&1; exec bash' + bash -c 'apt-get update >/dev/null 2>&1 && apt-get install -y --no-install-recommends fuse3 libfuse2t64 attr util-linux coreutils findutils git ca-certificates python3 >/dev/null 2>&1; exec bash'