diff --git a/javascript/extractor/lib/typescript/src/ast_extractor.ts b/javascript/extractor/lib/typescript/src/ast_extractor.ts index 3dbc8a52d641..a39395f12498 100644 --- a/javascript/extractor/lib/typescript/src/ast_extractor.ts +++ b/javascript/extractor/lib/typescript/src/ast_extractor.ts @@ -1,5 +1,4 @@ import * as ts from "./typescript"; -import { Project } from "./common"; /** * Interface that exposes some internal properties we rely on, as well as @@ -27,27 +26,6 @@ export interface Token { text: string; } -function hasOwnProperty(o: object, p: string) { - return o && Object.prototype.hasOwnProperty.call(o, p); -} - -// build our own copy of `ts.SyntaxKind` without `First*`/`Last*` markers -let SyntaxKind: { [id: number]: string } = []; -for (let p in ts.SyntaxKind) { - if (!hasOwnProperty(ts.SyntaxKind, p)) { - continue; - } - // skip numeric indices - if (+p === +p) { - continue; - } - // skip `First*`/`Last*` - if (p.substring(0, 5) === "First" || p.substring(0, 4) === "Last") { - continue; - } - SyntaxKind[ts.SyntaxKind[p] as any] = p; -} - let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; /** @@ -61,18 +39,7 @@ function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) { visit(ast); } -function tryGetTypeOfNode(typeChecker: ts.TypeChecker, node: AugmentedNode): ts.Type | null { - try { - return typeChecker.getTypeAtLocation(node); - } catch (e) { - let sourceFile = node.getSourceFile(); - let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.pos); - console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line + 1}:${character + 1}`); - return null; - } -} - -export function augmentAst(ast: AugmentedSourceFile, code: string, project: Project | null) { +export function augmentAst(ast: AugmentedSourceFile, code: string) { ast.$lineStarts = ast.getLineStarts(); /** @@ -164,10 +131,9 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj } if (ts.isVariableDeclarationList(node as any)) { - let tz = ts as any; - if (typeof tz.isLet === "function" && tz.isLet(node) || (node.flags & ts.NodeFlags.Let)) { + if (node.flags & ts.NodeFlags.Let) { node.$declarationKind = "let"; - } else if (typeof tz.isConst === "function" && tz.isConst(node) || (node.flags & ts.NodeFlags.Const)) { + } else if (node.flags & ts.NodeFlags.Const) { node.$declarationKind = "const"; } else { node.$declarationKind = "var"; diff --git a/javascript/extractor/lib/typescript/src/common.ts b/javascript/extractor/lib/typescript/src/common.ts deleted file mode 100644 index 9276a567bb08..000000000000 --- a/javascript/extractor/lib/typescript/src/common.ts +++ /dev/null @@ -1,35 +0,0 @@ -import * as ts from "./typescript"; - -export class Project { - public program: ts.Program = null; - private host: ts.CompilerHost; - - constructor( - public tsConfig: string, - public config: ts.ParsedCommandLine, - public packageEntryPoints: Map) { - - let host = ts.createCompilerHost(config.options, true); - host.trace = undefined; // Disable tracing which would otherwise go to standard out - this.host = host; - } - - public unload(): void { - this.program = null; - } - - public load(): void { - const { config, host } = this; - this.program = ts.createProgram(config.fileNames, config.options, host); - } - - /** - * Discards the old compiler instance and starts a new one. - */ - public reload(): void { - // Ensure all references to the old compiler instance - // are cleared before calling `createProgram`. - this.unload(); - this.load(); - } -} diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index 0630213c7431..3d6b10561f3b 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -36,9 +36,6 @@ import * as readline from "readline"; import * as ts from "./typescript"; import * as ast_extractor from "./ast_extractor"; -import { Project } from "./common"; -import { VirtualSourceRoot } from "./virtual_source_root"; - // Remove limit on stack trace depth. Error.stackTraceLimit = Infinity; @@ -46,13 +43,6 @@ interface ParseCommand { command: "parse"; filename: string; } -interface LoadCommand { - tsConfig: string; - sourceRoot: string | null; - virtualSourceRoot: string | null; - packageEntryPoints: [string, string][]; - packageJsonFiles: [string, string][]; -} interface ResetCommand { command: "reset"; } @@ -70,93 +60,15 @@ type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand | /** The state to be shared between commands. */ class State { - public project: Project = null; - /** List of files that have been requested. */ public pendingFiles: string[] = []; public pendingFileIndex = 0; /** Next response to be delivered. */ public pendingResponse: string = null; - - /** Map from `package.json` files to their contents. */ - public parsedPackageJson = new Map(); - - /** Map from `package.json` files to the file referenced in its `types` or `typings` field. */ - public packageTypings = new Map(); - - /** Map from file path to the enclosing `package.json` file, if any. Will not traverse outside node_modules. */ - public enclosingPackageJson = new Map(); } let state = new State(); -const reloadMemoryThresholdMb = getEnvironmentVariable("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", Number, 1000); - -function getPackageJson(file: string): any { - let cache = state.parsedPackageJson; - if (cache.has(file)) return cache.get(file); - let result = getPackageJsonRaw(file); - cache.set(file, result); - return result; -} - -function getPackageJsonRaw(file: string): any { - if (!ts.sys.fileExists(file)) return undefined; - try { - let json = JSON.parse(ts.sys.readFile(file)); - if (typeof json !== 'object') return undefined; - return json; - } catch (e) { - return undefined; - } -} - -function getPackageTypings(file: string): string | undefined { - let cache = state.packageTypings; - if (cache.has(file)) return cache.get(file); - let result = getPackageTypingsRaw(file); - cache.set(file, result); - return result; -} - -function getPackageTypingsRaw(packageJsonFile: string): string | undefined { - let json = getPackageJson(packageJsonFile); - if (json == null) return undefined; - let typings = json.types || json.typings; // "types" and "typings" are aliases - if (typeof typings !== 'string') return undefined; - let absolutePath = pathlib.join(pathlib.dirname(packageJsonFile), typings); - if (ts.sys.directoryExists(absolutePath)) { - absolutePath = pathlib.join(absolutePath, 'index.d.ts'); - } else if (!absolutePath.endsWith('.ts')) { - absolutePath += '.d.ts'; - } - if (!ts.sys.fileExists(absolutePath)) return undefined; - return ts.sys.resolvePath(absolutePath); -} - -function getEnclosingPackageJson(file: string): string | undefined { - let cache = state.packageTypings; - if (cache.has(file)) return cache.get(file); - let result = getEnclosingPackageJsonRaw(file); - cache.set(file, result); - return result; -} - -function getEnclosingPackageJsonRaw(file: string): string | undefined { - let packageJson = pathlib.join(file, 'package.json'); - if (ts.sys.fileExists(packageJson)) { - return packageJson; - } - if (pathlib.basename(file) === 'node_modules') { - return undefined; - } - let dirname = pathlib.dirname(file); - if (dirname.length < file.length) { - return getEnclosingPackageJson(dirname); - } - return undefined; -} - /** * Debugging method for finding cycles in the TypeScript AST. Should not be used in production. * @@ -330,7 +242,6 @@ function extractFile(filename: string): string { function prepareNextFile() { if (state.pendingResponse != null) return; if (state.pendingFileIndex < state.pendingFiles.length) { - checkMemoryUsage(); let nextFilename = state.pendingFiles[state.pendingFileIndex]; state.pendingResponse = extractFile(nextFilename); } @@ -368,13 +279,12 @@ function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolea } /** - * Gets the AST and source code for the given file, either from - * an already-open project, or by parsing the file. + * Gets the AST and source code by parsing the given file. */ function getAstForFile(filename: string): ts.SourceFile { let { ast, code } = parseSingleFile(filename); if (ast != null && isExtractableSourceFile(ast)) { - ast_extractor.augmentAst(ast, code, null); + ast_extractor.augmentAst(ast, code); } return ast; } @@ -401,7 +311,6 @@ function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } // parse `filename` with minimial checking let compilerOptions: ts.CompilerOptions = { experimentalDecorators: true, - experimentalAsyncFunctions: true, jsx: ts.JsxEmit.Preserve, noResolve: true, }; @@ -413,98 +322,6 @@ function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } return { ast, code }; } -/** - * Matches a path segment referencing a package in a node_modules folder, and extracts - * two capture groups: the package name, and the relative path in the package. - * - * For example `lib/node_modules/@foo/bar/src/index.js` extracts the capture groups [`@foo/bar`, `src/index.js`]. - */ -const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/; - -interface LoadedConfig { - config: ts.ParsedCommandLine; - basePath: string; - packageEntryPoints: Map; - packageJsonFiles: Map; - virtualSourceRoot: VirtualSourceRoot; - ownFiles: string[]; -} - -function loadTsConfig(command: LoadCommand): LoadedConfig { - let tsConfig = ts.readConfigFile(command.tsConfig, ts.sys.readFile); - let basePath = pathlib.dirname(command.tsConfig); - - let packageEntryPoints = new Map(command.packageEntryPoints); - let packageJsonFiles = new Map(command.packageJsonFiles); - let virtualSourceRoot = new VirtualSourceRoot(command.sourceRoot, command.virtualSourceRoot); - - /** - * Rewrites path segments of form `node_modules/PACK/suffix` to be relative to - * the location of package PACK in the source tree, if it exists. - */ - function redirectNodeModulesPath(path: string) { - let nodeModulesMatch = nodeModulesRex.exec(path); - if (nodeModulesMatch == null) return null; - let packageName = nodeModulesMatch[1]; - let packageJsonFile = packageJsonFiles.get(packageName); - if (packageJsonFile == null) return null; - let packageDir = pathlib.dirname(packageJsonFile); - let suffix = nodeModulesMatch[2]; - let finalPath = pathlib.join(packageDir, suffix); - if (!ts.sys.fileExists(finalPath)) return null; - return finalPath; - } - - /** - * Create the host passed to the tsconfig.json parser. - * - * We override its file system access in case there is an "extends" - * clause pointing into "./node_modules", which must be redirected to - * the location of an installed package or a checked-in package. - */ - let parseConfigHost: ts.ParseConfigHost = { - useCaseSensitiveFileNames: true, - readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => { - // Perform the glob matching in both real and virtual source roots. - let exclusions = excludes == null ? [] : [...excludes]; - if (virtualSourceRoot.virtualSourceRoot != null) { - // qltest puts the virtual source root inside the real source root (.testproj). - // Make sure we don't find files inside the virtual source root in this pass. - exclusions.push(virtualSourceRoot.virtualSourceRoot); - } - let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth) - let virtualDir = virtualSourceRoot.toVirtualPath(rootDir); - if (virtualDir == null) { - return originalResults; - } - // Make sure glob matching does not to discover anything in node_modules. - let virtualExclusions = excludes == null ? [] : [...excludes]; - virtualExclusions.push('**/node_modules/**/*'); - let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth) - return [...originalResults, ...virtualResults]; - }, - fileExists: (path: string) => { - return ts.sys.fileExists(path) - || virtualSourceRoot.toVirtualPathIfFileExists(path) != null - || redirectNodeModulesPath(path) != null; - }, - readFile: (path: string) => { - if (!ts.sys.fileExists(path)) { - let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path); - if (virtualPath != null) return ts.sys.readFile(virtualPath); - virtualPath = redirectNodeModulesPath(path); - if (virtualPath != null) return ts.sys.readFile(virtualPath); - } - return ts.sys.readFile(path); - } - }; - let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath); - - let ownFiles = config.fileNames.map(file => pathlib.resolve(file)); - - return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles }; -} - function handleResetCommand(command: ResetCommand) { reset(); console.log(JSON.stringify({ @@ -533,37 +350,6 @@ function reset() { state = new State(); } -function getEnvironmentVariable(name: string, parse: (x: string) => T, defaultValue: T) { - let value = process.env[name]; - return value != null ? parse(value) : defaultValue; -} - -/** - * Whether the memory usage was last observed to be above the threshold for restarting the TypeScript compiler. - * - * This is to prevent repeatedly restarting the compiler if the GC does not immediately bring us below the - * threshold again. - */ -let hasReloadedSinceExceedingThreshold = false; - -/** - * If memory usage has moved above a the threshold, reboot the TypeScript compiler instance. - * - * Make sure to call this only when stdout has been flushed. - */ -function checkMemoryUsage() { - let bytesUsed = process.memoryUsage().heapUsed; - let megabytesUsed = bytesUsed / 1000000; - if (!hasReloadedSinceExceedingThreshold && megabytesUsed > reloadMemoryThresholdMb && state.project != null) { - console.warn('Restarting TypeScript compiler due to memory usage'); - state.project.reload(); - hasReloadedSinceExceedingThreshold = true; - } - else if (hasReloadedSinceExceedingThreshold && megabytesUsed < reloadMemoryThresholdMb) { - hasReloadedSinceExceedingThreshold = false; - } -} - function runReadLineInterface() { reset(); let rl = readline.createInterface({ input: process.stdin, output: process.stdout }); diff --git a/javascript/extractor/lib/typescript/src/virtual_source_root.ts b/javascript/extractor/lib/typescript/src/virtual_source_root.ts deleted file mode 100644 index 6ca9a580b566..000000000000 --- a/javascript/extractor/lib/typescript/src/virtual_source_root.ts +++ /dev/null @@ -1,69 +0,0 @@ -import * as pathlib from "path"; -import * as ts from "./typescript"; - -/** - * Mapping from the real source root to the virtual source root, - * a directory whose folder structure mirrors the real source root, but with `node_modules` installed. - */ -export class VirtualSourceRoot { - constructor( - public sourceRoot: string | null, - - /** - * Directory whose folder structure mirrors the real source root, but with `node_modules` installed, - * or undefined if no virtual source root exists. - */ - public virtualSourceRoot: string | null, - ) {} - - private static translate(oldRoot: string, newRoot: string, path: string) { - if (!oldRoot || !newRoot) return null; - let relative = pathlib.relative(oldRoot, path); - if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null; - return pathlib.join(newRoot, relative); - } - - /** - * Maps a path under the real source root to the corresponding path in the virtual source root. - * - * Returns `null` for paths already in the virtual source root. - */ - public toVirtualPath(path: string) { - let { virtualSourceRoot } = this; - if (path.startsWith(virtualSourceRoot)) { - // 'qltest' creates a virtual source root inside the real source root. - // Make sure such files don't appear to be inside the real source root. - return null; - } - return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path); - } - - /** - * Maps a path under the virtual source root to the corresponding path in the real source root. - */ - public fromVirtualPath(path: string) { - return VirtualSourceRoot.translate(this.virtualSourceRoot, this.sourceRoot, path); - } - - /** - * Maps a path under the real source root to the corresponding path in the virtual source root. - */ - public toVirtualPathIfFileExists(path: string) { - let virtualPath = this.toVirtualPath(path); - if (virtualPath != null && ts.sys.fileExists(virtualPath)) { - return virtualPath; - } - return null; - } - - /** - * Maps a path under the real source root to the corresponding path in the virtual source root. - */ - public toVirtualPathIfDirectoryExists(path: string) { - let virtualPath = this.toVirtualPath(path); - if (virtualPath != null && ts.sys.directoryExists(virtualPath)) { - return virtualPath; - } - return null; - } -}