diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index ae63510..9487949 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -1,9 +1,9 @@ import { FSLike, fdir } from "../src/index"; import fs from "fs"; import mock from "mock-fs"; -import { test, beforeEach, TestContext, vi } from "vitest"; +import { test, describe, beforeEach, afterEach, TestContext, vi } from "vitest"; import path, { sep } from "path"; -import { convertSlashes } from "../src/utils"; +import { convertSlashes, normalizePath } from "../src/utils"; import picomatch from "picomatch"; import { apiTypes, APITypes, cwd, restricted, root } from "./utils"; @@ -45,6 +45,7 @@ for (const type of apiTypes) { includeBasePath: true, suppressErrors: false, }).crawl("__tests__"); + const files = await api[type](); t.expect(files.every((file) => file.startsWith("__tests__"))).toBeTruthy(); }); @@ -348,13 +349,152 @@ for (const type of apiTypes) { test(`[${type}] crawl all files and invert path separator`, async (t) => { const api = new fdir() .withErrors() + .withBasePath() .withPathSeparator(sep === "/" ? "\\" : "/") .crawl("node_modules"); const files = await api[type](); + t.expect(files).not.toHaveLength(0); t.expect(files.every((f) => !f.includes(sep))).toBeTruthy(); }); + for (const separator of ["/", "\\"] as const) { + describe(`[${type}] output paths with the "${separator}" separator`, () => { + const inSeparator = (paths: string[]) => + paths.map((p) => convertSlashes(p, separator)).sort(); + + beforeEach(() => { + mock({ "/separators/a/b": { file: "" }, "/separators/c": "" }); + }); + + afterEach(() => { + mock.restore(); + }); + + test(`with base paths`, async (t) => { + const api = new fdir() + .withErrors() + .withBasePath() + .withPathSeparator(separator) + .crawl("/separators"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual( + inSeparator(["/separators/a/b/file", "/separators/c"]) + ); + }); + + test(`with relative paths`, async (t) => { + const api = new fdir() + .withErrors() + .withRelativePaths() + .withPathSeparator(separator) + .crawl("/separators"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual(inSeparator(["a/b/file", "c"])); + }); + + test(`with directories`, async (t) => { + const api = new fdir() + .withErrors() + .withDirs() + .withPathSeparator(separator) + .crawl("/separators"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual( + inSeparator([ + "/separators/", + "/separators/a/", + "/separators/a/b/", + "c", + "file", + ]) + ); + }); + + test(`with directories and relative paths`, async (t) => { + const api = new fdir() + .withErrors() + .withDirs() + .withRelativePaths() + .withPathSeparator(separator) + .crawl("/separators"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual( + inSeparator([".", "a/", "a/b/", "a/b/file", "c"]) + ); + }); + + test(`in groups`, async (t) => { + const api = new fdir() + .withErrors() + .withBasePath() + .withPathSeparator(separator) + .group() + .crawl("/separators"); + const groups = await api[type](); + t.expect(groups.map((group) => group.directory).sort()).toStrictEqual( + inSeparator(["/separators/", "/separators/a/", "/separators/a/b/"]) + ); + t.expect(groups.flatMap((group) => group.files).sort()).toStrictEqual( + inSeparator(["/separators/a/b/file", "/separators/c"]) + ); + }); + + test(`given to exclude`, async (t) => { + const excluded: string[] = []; + const api = new fdir() + .withErrors() + .withPathSeparator(separator) + .exclude((_name, path) => { + excluded.push(path); + return false; + }) + .crawl("/separators"); + await api[type](); + t.expect(excluded.sort()).toStrictEqual( + inSeparator(["/separators/a/", "/separators/a/b/"]) + ); + }); + }); + } + + test.skipIf(sep === "\\")( + `[${type}] crawl names that contain "\\" with the "\\" separator`, + async (t) => { + mock({ "/separators": { "a\\b": "", "c\\d": { file: "" } } }); + const api = new fdir() + .withErrors() + .withBasePath() + .withPathSeparator("\\") + .crawl("/separators"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual([ + "\\separators\\a\\b", + "\\separators\\c\\d\\file", + ]); + mock.restore(); + } + ); + + for (const separator of ["/", "\\"] as const) { + test.skipIf(sep === "\\")( + `[${type}] crawl a root whose name ends with "\\" with the "${separator}" separator`, + async (t) => { + mock({ + "literal\\": { "wanted.txt": "" }, + literal: { "decoy.txt": "" }, + }); + const api = new fdir() + .withErrors() + .withPathSeparator(separator) + .crawl("literal\\"); + const files = await api[type](); + t.expect(files).toStrictEqual(["wanted.txt"]); + mock.restore(); + } + ); + } + test(`[${type}] crawl files that match using a custom glob`, async (t) => { const globFunction = vi.fn((glob: string | string[]) => { return (test: string): boolean => test.endsWith(".js"); @@ -533,7 +673,5 @@ test(`do not convert \\\\ to \\`, async (t) => { }); test(`do not add a separator if path already ends with one`, async (t) => { - t.expect(normalizePath("some-root/", { pathSeparator: "\\" })).toBe( - "some-root\\" - ); + t.expect(normalizePath("some-root/", {})).toBe(`some-root${sep}`); }); diff --git a/__tests__/symlinks.test.ts b/__tests__/symlinks.test.ts index 00795e7..274178c 100644 --- a/__tests__/symlinks.test.ts +++ b/__tests__/symlinks.test.ts @@ -1,7 +1,8 @@ import { afterAll, beforeAll, beforeEach, describe, test } from "vitest"; import { apiTypes, normalize, root } from "./utils"; import mock from "mock-fs"; -import { fdir, Options } from "../src"; +import { fdir, Options, PathSeparator } from "../src"; +import { convertSlashes } from "../src/utils"; import path from "path"; const fsWithRelativeSymlinks = { @@ -16,9 +17,6 @@ const fsWithRelativeSymlinks = { fileSymlink: mock.symlink({ path: "../../../../other-relative/dir/file-2", }), - fileSymlink2: mock.symlink({ - path: "../../../../other-relative/dir/file-3", - }), dirSymlink: mock.symlink({ path: "../../../../sym-relative/linked", }), @@ -133,9 +131,6 @@ const mockFs = { fileSymlink: mock.symlink({ path: "/other/dir/file-2", }), - fileSymlink2: mock.symlink({ - path: "/other/dir/file-3", - }), dirSymlink: mock.symlink({ path: "/sym/linked", }), @@ -376,7 +371,7 @@ for (const type of apiTypes) { const api = new fdir().withErrors().crawl("/some/dir"); const files = await api[type](); t.expect(files.sort()).toStrictEqual( - normalize(["dirSymlink", "fileSymlink", "fileSymlink2"]) + normalize(["dirSymlink", "fileSymlink"]) ); }); @@ -404,7 +399,193 @@ for (const type of apiTypes) { }, { timeout: 1000 } ); + + // Each crawl gives the files of its native-separator crawl, in the other separator + const separator = path.sep === "/" ? "\\" : "/"; + const crawls: Record< + string, + (api: fdir, separator: PathSeparator) => ReturnType + > = { + "resolve symlinks": (api) => api.withSymlinks().crawl("/some/dir"), + "resolve symlinks (real paths: false)": (api) => + api.withSymlinks({ resolvePaths: false }).crawl("/some/dir"), + "resolve symlinks (real paths: false, relative paths: true)": (api) => + api + .withSymlinks({ resolvePaths: false }) + .withRelativePaths() + .crawl("/some/dir"), + "resolve symlinks (real paths: true, relative paths: true)": (api) => + api.withSymlinks().withRelativePaths().crawl("./relative/dir"), + "resolve symlinks (exclude /sym/linked/)": (api, separator) => + api + .withSymlinks() + .exclude( + (_name, path) => + path === + convertSlashes(resolveSymlinkRoot("/sym/linked/"), separator) + ) + .crawl("/some/dir"), + "resolve symlinks (exclude /some/dir/dirSymlink/, real paths: false)": ( + api, + separator + ) => + api + .withSymlinks({ resolvePaths: false }) + .exclude( + (_name, path) => + path === + convertSlashes( + resolveSymlinkRoot("/some/dir/dirSymlink/"), + separator + ) + ) + .crawl("/some/dir"), + "resolve recursive symlinks": (api) => + api.withSymlinks().crawl("/recursive"), + "resolve recursive symlinks (real paths: false)": (api) => + api.withSymlinks({ resolvePaths: false }).crawl("/recursive"), + "resolve recursive symlinks (real paths: true, relative paths: true)": ( + api + ) => api.withSymlinks().withRelativePaths().crawl("./recursive"), + "resolve recursive symlinks (real paths: false, relative paths: true)": ( + api + ) => + api + .withSymlinks({ resolvePaths: false }) + .withRelativePaths() + .crawl("./recursive"), + }; + + for (const [name, crawl] of Object.entries(crawls)) { + test(`${name} with the "${separator}" separator`, async (t) => { + const nativeFiles = await crawl(new fdir().withErrors(), path.sep)[ + type + ](); + const files = await crawl( + new fdir().withErrors().withPathSeparator(separator), + separator + )[type](); + t.expect(files).not.toHaveLength(0); + t.expect(files.sort()).toStrictEqual( + nativeFiles.map((file) => convertSlashes(file, separator)).sort() + ); + }); + } }); + + describe(`${type}: a symlink whose target is missing`, () => { + beforeAll(() => { + mock({ + "/elsewhere": { + file: "file contents", + }, + "/dangling": { + kept: mock.symlink({ + path: "/elsewhere/file", + }), + broken: mock.symlink({ + path: "/elsewhere/missing", + }), + }, + }); + }); + + afterAll(() => { + mock.restore(); + }); + + test(`is listed by its name when symlinks are not resolved`, async (t) => { + const api = new fdir().withErrors().crawl("/dangling"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual(normalize(["broken", "kept"])); + }); + + test(`is skipped when symlinks are resolved`, async (t) => { + const api = new fdir().withSymlinks().crawl("/dangling"); + const files = await api[type](); + t.expect(files).toStrictEqual(normalize(["/elsewhere/file"])); + }); + + test(`is reported when symlinks are resolved with errors`, async (t) => { + const api = new fdir().withErrors().withSymlinks().crawl("/dangling"); + await t + .expect(Promise.resolve().then(() => api[type]())) + .rejects.toThrow(/ENOENT/); + }); + }); + + describe.skipIf(path.sep === "\\")( + `${type}: symlinks to a directory whose name contains "\\"`, + () => { + beforeAll(() => { + mock({ + "/backslash": { + "dir\\name": { wanted: "" }, + dir: { name: { decoy: "" } }, + root: { + link: mock.symlink({ path: "/backslash/dir\\name" }), + "file-link": mock.symlink({ + path: "/backslash/dir\\name/wanted", + }), + }, + }, + }); + }); + + afterAll(() => { + mock.restore(); + }); + + for (const separator of ["/", "\\"] as const) { + // Only the separators change: the backslash in the name stays + const inSeparator = (paths: string[]) => + paths.map((p) => p.split("/").join(separator)).sort(); + + test(`resolve them with the "${separator}" separator`, async (t) => { + const api = new fdir() + .withErrors() + .withPathSeparator(separator) + .withSymlinks() + .crawl("/backslash/root"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual( + inSeparator([ + "/backslash/dir\\name/wanted", + "/backslash/dir\\name/wanted", + ]) + ); + }); + + test(`resolve them (real paths: false) with the "${separator}" separator`, async (t) => { + const api = new fdir() + .withErrors() + .withPathSeparator(separator) + .withSymlinks({ resolvePaths: false }) + .crawl("/backslash/root"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual( + inSeparator([ + "/backslash/root/file-link", + "/backslash/root/link/wanted", + ]) + ); + }); + + test(`resolve them (relative paths: true) with the "${separator}" separator`, async (t) => { + const api = new fdir() + .withErrors() + .withPathSeparator(separator) + .withSymlinks() + .withRelativePaths() + .crawl("/backslash/root"); + const files = await api[type](); + t.expect(files.sort()).toStrictEqual( + inSeparator(["../dir\\name/wanted", "../dir\\name/wanted"]) + ); + }); + } + } + ); } function resolveSymlinkRoot(p: string) { diff --git a/src/api/functions/join-path.ts b/src/api/functions/join-path.ts index d103cb2..74cd69e 100644 --- a/src/api/functions/join-path.ts +++ b/src/api/functions/join-path.ts @@ -1,18 +1,24 @@ import { relative } from "path"; import { Options, PathSeparator } from "../../types"; -import { convertSlashes } from "../../utils"; +import { toOutputPath } from "../../utils"; export function joinPathWithBasePath(filename: string, directoryPath: string) { return directoryPath + filename; } function joinPathWithRelativePath(root: string, options: Options) { - return function (filename: string, directoryPath: string) { + return function ( + filename: string, + outputPath: string, + directoryPath: string + ) { const sameRoot = directoryPath.startsWith(root); - if (sameRoot) return directoryPath.slice(root.length) + filename; + // The output path is the native path with other separators, so the root + // ends at the same index in both + if (sameRoot) return outputPath.slice(root.length) + filename; else return ( - convertSlashes(relative(root, directoryPath), options.pathSeparator) + + toOutputPath(relative(root, directoryPath), options.pathSeparator) + options.pathSeparator + filename ); @@ -31,8 +37,11 @@ export function joinDirectoryPath( return directoryPath + filename + separator; } +// Joins a name to its directory's output path. The native path serves +// `relative()`, when a resolved symlink leads outside the root. export type JoinPathFunction = ( filename: string, + outputPath: string, directoryPath: string ) => string; diff --git a/src/api/functions/resolve-symlink.ts b/src/api/functions/resolve-symlink.ts index b23cdb2..4480ce0 100644 --- a/src/api/functions/resolve-symlink.ts +++ b/src/api/functions/resolve-symlink.ts @@ -1,6 +1,6 @@ import type { Stats } from "fs"; import { WalkerState, Options } from "../../types"; -import { dirname } from "path"; +import { dirname, sep } from "path"; export type ResolveSymlinkFunction = ( path: string, @@ -89,5 +89,5 @@ function isRecursive(path: string, resolved: string, state: WalkerState) { } function isRecursiveUsingRealPaths(resolved: string, state: WalkerState) { - return state.visited.includes(resolved + state.options.pathSeparator); + return state.visited.includes(resolved + sep); } diff --git a/src/api/functions/walk-directory.ts b/src/api/functions/walk-directory.ts index f2b9d36..5ee4616 100644 --- a/src/api/functions/walk-directory.ts +++ b/src/api/functions/walk-directory.ts @@ -5,8 +5,14 @@ export type WalkDirectoryFunction = ( state: WalkerState, crawlPath: string, directoryPath: string, + outputPath: string, depth: number, - callback: (entries: Dirent[], directoryPath: string, depth: number) => void + callback: ( + entries: Dirent[], + directoryPath: string, + outputPath: string, + depth: number + ) => void ) => void; const readdirOpts = { withFileTypes: true } as const; @@ -15,6 +21,7 @@ const walkAsync: WalkDirectoryFunction = ( state, crawlPath, directoryPath, + outputPath, currentDepth, callback ) => { @@ -30,7 +37,7 @@ const walkAsync: WalkDirectoryFunction = ( // Perf: Node >= 10 introduced withFileTypes that helps us // skip an extra fs.stat call. fs.readdir(crawlPath || ".", readdirOpts, (error, entries = []) => { - callback(entries, directoryPath, currentDepth); + callback(entries, directoryPath, outputPath, currentDepth); state.queue.dequeue(state.options.suppressErrors ? null : error, state); }); @@ -40,6 +47,7 @@ const walkSync: WalkDirectoryFunction = ( state, crawlPath, directoryPath, + outputPath, currentDepth, callback ) => { @@ -54,7 +62,7 @@ const walkSync: WalkDirectoryFunction = ( } catch (e) { if (!state.options.suppressErrors) throw e; } - callback(entries, directoryPath, currentDepth); + callback(entries, directoryPath, outputPath, currentDepth); }; export function build(isSynchronous: boolean): WalkDirectoryFunction { diff --git a/src/api/walker.ts b/src/api/walker.ts index e6edc0d..f8d92ac 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -1,5 +1,5 @@ -import { basename, dirname } from "path"; -import { isRootDirectory, normalizePath } from "../utils"; +import { basename, dirname, sep } from "path"; +import { isRootDirectory, normalizePath, toOutputPath } from "../utils"; import { ResultCallback, WalkerState, Options } from "../types"; import * as joinPath from "./functions/join-path"; import * as pushDirectory from "./functions/push-directory"; @@ -16,8 +16,15 @@ import { Output } from "../types"; import { Counter } from "./counter"; import { Aborter } from "./aborter"; +/* + * The walker hands the file system native paths and gives the custom separator + * only to what it outputs: converting an output path back would read a name + * that contains the other separator as two names. Each directory carries both + * forms, which are the same string when the separators match. + */ export class Walker { private readonly root: string; + private readonly outputRoot: string; private readonly isSynchronous: boolean; private readonly state: WalkerState; private readonly joinPath: joinPath.JoinPathFunction; @@ -38,6 +45,7 @@ export class Walker { this.callbackInvoker = invokeCallback.build(options, this.isSynchronous); this.root = normalizePath(root, options); + this.outputRoot = toOutputPath(this.root, options.pathSeparator); this.state = { root: isRootDirectory(this.root) ? this.root : this.root.slice(0, -1), // Perf: we explicitly tell the compiler to optimize for String arrays @@ -60,7 +68,7 @@ export class Walker { * by the javascript engine so there's no function call overhead (in most cases). */ this.joinPath = joinPath.build(this.root, options); - this.pushDirectory = pushDirectory.build(this.root, options); + this.pushDirectory = pushDirectory.build(this.outputRoot, options); this.pushFile = pushFile.build(options); this.getArray = getArray.build(options); this.groupFiles = groupFiles.build(options); @@ -69,18 +77,28 @@ export class Walker { } start(): TOutput | null { - this.pushDirectory(this.root, this.state.paths, this.state.options.filters); + this.pushDirectory( + this.outputRoot, + this.state.paths, + this.state.options.filters + ); this.walkDirectory( this.state, this.root, this.root, + this.outputRoot, this.state.options.maxDepth, this.walk ); return this.isSynchronous ? this.callbackInvoker(this.state, null) : null; } - private walk = (entries: Dirent[], directoryPath: string, depth: number) => { + private walk = ( + entries: Dirent[], + directoryPath: string, + outputPath: string, + depth: number + ) => { const { paths, options: { @@ -111,35 +129,38 @@ export class Walker { entry.isFile() || (entry.isSymbolicLink() && !resolveSymlinks && !excludeSymlinks) ) { - const filename = this.joinPath(entry.name, directoryPath); + const filename = this.joinPath(entry.name, outputPath, directoryPath); this.pushFile(filename, files, this.state.counts, filters); } else if (entry.isDirectory()) { - let path = joinPath.joinDirectoryPath( - entry.name, - directoryPath, - this.state.options.pathSeparator + let path = joinPath.joinDirectoryPath(entry.name, directoryPath, sep); + const output = + pathSeparator === sep + ? path + : joinPath.joinDirectoryPath(entry.name, outputPath, pathSeparator); + if (exclude && exclude(entry.name, output)) continue; + this.pushDirectory(output, paths, filters); + this.walkDirectory( + this.state, + path, + path, + output, + depth - 1, + this.walk ); - if (exclude && exclude(entry.name, path)) continue; - this.pushDirectory(path, paths, filters); - this.walkDirectory(this.state, path, path, depth - 1, this.walk); } else if (this.resolveSymlink && entry.isSymbolicLink()) { let path = joinPath.joinPathWithBasePath(entry.name, directoryPath); this.resolveSymlink(path, this.state, (stat, resolvedPath) => { if (stat.isDirectory()) { resolvedPath = normalizePath(resolvedPath, this.state.options); - if ( - exclude && - exclude( - entry.name, - useRealPaths ? resolvedPath : path + pathSeparator - ) - ) - return; + const directoryPath = useRealPaths ? resolvedPath : path + sep; + const outputPath = toOutputPath(directoryPath, pathSeparator); + if (exclude && exclude(entry.name, outputPath)) return; this.walkDirectory( this.state, resolvedPath, - useRealPaths ? resolvedPath : path + pathSeparator, + directoryPath, + outputPath, depth - 1, this.walk ); @@ -150,13 +171,17 @@ export class Walker { dirname(resolvedPath), this.state.options ); - resolvedPath = this.joinPath(filename, directoryPath); + resolvedPath = this.joinPath( + filename, + toOutputPath(directoryPath, pathSeparator), + directoryPath + ); this.pushFile(resolvedPath, files, this.state.counts, filters); } }); } } - this.groupFiles(this.state.groups, directoryPath, files); + this.groupFiles(this.state.groups, outputPath, files); }; } diff --git a/src/utils.ts b/src/utils.ts index 76905e5..6f43a81 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -17,20 +17,23 @@ export function convertSlashes(path: string, separator: PathSeparator) { return path.replace(SLASHES_REGEX, separator); } +// A native path in the output's separator, which leaves it as is when native +export function toOutputPath(path: string, separator: PathSeparator) { + return separator === sep ? path : convertSlashes(path, separator); +} + const WINDOWS_ROOT_DIR_REGEX = /^[a-z]:[\\/]$/i; export function isRootDirectory(path: string) { return path === "/" || WINDOWS_ROOT_DIR_REGEX.test(path); } +// Normalizes a path for the file system in the platform's own syntax, where a +// POSIX name can contain a backslash, and ends it with the native separator export function normalizePath( path: string, - options: { - resolvePaths?: boolean; - normalizePath?: boolean; - pathSeparator: PathSeparator; - } + options: { resolvePaths?: boolean; normalizePath?: boolean } ) { - const { resolvePaths, normalizePath, pathSeparator } = options; + const { resolvePaths, normalizePath } = options; const pathNeedsCleaning = (process.platform === "win32" && path.includes("/")) || path.startsWith("."); @@ -40,10 +43,6 @@ export function normalizePath( if (path === ".") return ""; - const needsSeperator = - path[path.length - 1] !== "/" && path[path.length - 1] !== "\\"; - return convertSlashes( - needsSeperator ? path + pathSeparator : path, - pathSeparator - ); + // On Windows a "/" gets the path cleaned above, so only `sep` can end it + return path[path.length - 1] === sep ? path : path + sep; }