Skip to content

JS: TypeScript extractor cleanup #22860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paldepind merged 10 commits into from
Oct 21, 2026
Merged

JS: TypeScript extractor cleanup #22860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3f67c7f
JS: Remove unused TypeScript package-resolution caches
paldepind Oct 21, 2026
b5c9e56
JS: Remove unused TypeScript project configuration loading
paldepind Oct 21, 2026
b0062cf
JS: Remove ineffective TypeScript project memory checks
paldepind Oct 21, 2026
04b4861
JS: Remove unused TypeScript Project abstraction
paldepind Oct 21, 2026
17c8e4a
JS: Remove unused TypeScript AST cycle debugger
paldepind Oct 21, 2026
9416214
JS: Remove unused TypeScript syntax-kind lookup
paldepind Oct 21, 2026
51531b3
JS: Remove unused TypeScript type-checker helper
paldepind Oct 21, 2026
ad9e569
JS: Remove obsolete `experimentalAsyncFunctions` option
paldepind Oct 21, 2026
53bdb08
JS: Simplify TypeScript declaration-kind detection
paldepind Oct 21, 2026
0610208
Revert "JS: Remove unused TypeScript AST cycle debugger"
paldepind Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 3 additions & 37 deletions javascript/extractor/lib/typescript/src/ast_extractor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import * as ts from "./typescript";
import { Project } from "./common";

/**
* Interface that exposes some internal properties we rely on, as well as
Expand Down Expand Up @@ -27,27 +26,6 @@ export interface Token {
text: string;
}

function hasOwnProperty(o: object, p: string) {
return o && Object.prototype.hasOwnProperty.call(o, p);
}

// build our own copy of `ts.SyntaxKind` without `First*`/`Last*` markers
let SyntaxKind: { [id: number]: string } = [];
for (let p in ts.SyntaxKind) {
if (!hasOwnProperty(ts.SyntaxKind, p)) {
continue;
}
// skip numeric indices
if (+p === +p) {
continue;
}
// skip `First*`/`Last*`
if (p.substring(0, 5) === "First" || p.substring(0, 4) === "Last") {
continue;
}
SyntaxKind[ts.SyntaxKind[p] as any] = p;
}

let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g;

/**
Expand All @@ -61,18 +39,7 @@ function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) {
visit(ast);
}

function tryGetTypeOfNode(typeChecker: ts.TypeChecker, node: AugmentedNode): ts.Type | null {
try {
return typeChecker.getTypeAtLocation(node);
} catch (e) {
let sourceFile = node.getSourceFile();
let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.pos);
console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line + 1}:${character + 1}`);
return null;
}
}

export function augmentAst(ast: AugmentedSourceFile, code: string, project: Project | null) {
export function augmentAst(ast: AugmentedSourceFile, code: string) {
ast.$lineStarts = ast.getLineStarts();

/**
Expand Down Expand Up @@ -164,10 +131,9 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj
}

if (ts.isVariableDeclarationList(node as any)) {
let tz = ts as any;
if (typeof tz.isLet === "function" && tz.isLet(node) || (node.flags & ts.NodeFlags.Let)) {
if (node.flags & ts.NodeFlags.Let) {
node.$declarationKind = "let";
} else if (typeof tz.isConst === "function" && tz.isConst(node) || (node.flags & ts.NodeFlags.Const)) {
} else if (node.flags & ts.NodeFlags.Const) {
node.$declarationKind = "const";
} else {
node.$declarationKind = "var";
Expand Down
35 changes: 0 additions & 35 deletions javascript/extractor/lib/typescript/src/common.ts
View file
Open in desktop

This file was deleted.

218 changes: 2 additions & 216 deletions javascript/extractor/lib/typescript/src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,23 +36,13 @@ import * as readline from "readline";
import * as ts from "./typescript";
import * as ast_extractor from "./ast_extractor";

import { Project } from "./common";
import { VirtualSourceRoot } from "./virtual_source_root";

// Remove limit on stack trace depth.
Error.stackTraceLimit = Infinity;

interface ParseCommand {
command: "parse";
filename: string;
}
interface LoadCommand {
tsConfig: string;
sourceRoot: string | null;
virtualSourceRoot: string | null;
packageEntryPoints: [string, string][];
packageJsonFiles: [string, string][];
}
interface ResetCommand {
command: "reset";
}
Expand All @@ -70,93 +60,15 @@ type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand |

/** The state to be shared between commands. */
class State {
public project: Project = null;

/** List of files that have been requested. */
public pendingFiles: string[] = [];
public pendingFileIndex = 0;

/** Next response to be delivered. */
public pendingResponse: string = null;

/** Map from `package.json` files to their contents. */
public parsedPackageJson = new Map<string, any>();

/** Map from `package.json` files to the file referenced in its `types` or `typings` field. */
public packageTypings = new Map<string, string | undefined>();

/** Map from file path to the enclosing `package.json` file, if any. Will not traverse outside node_modules. */
public enclosingPackageJson = new Map<string, string | undefined>();
}
let state = new State();

const reloadMemoryThresholdMb = getEnvironmentVariable("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", Number, 1000);

function getPackageJson(file: string): any {
let cache = state.parsedPackageJson;
if (cache.has(file)) return cache.get(file);
let result = getPackageJsonRaw(file);
cache.set(file, result);
return result;
}

function getPackageJsonRaw(file: string): any {
if (!ts.sys.fileExists(file)) return undefined;
try {
let json = JSON.parse(ts.sys.readFile(file));
if (typeof json !== 'object') return undefined;
return json;
} catch (e) {
return undefined;
}
}

function getPackageTypings(file: string): string | undefined {
let cache = state.packageTypings;
if (cache.has(file)) return cache.get(file);
let result = getPackageTypingsRaw(file);
cache.set(file, result);
return result;
}

function getPackageTypingsRaw(packageJsonFile: string): string | undefined {
let json = getPackageJson(packageJsonFile);
if (json == null) return undefined;
let typings = json.types || json.typings; // "types" and "typings" are aliases
if (typeof typings !== 'string') return undefined;
let absolutePath = pathlib.join(pathlib.dirname(packageJsonFile), typings);
if (ts.sys.directoryExists(absolutePath)) {
absolutePath = pathlib.join(absolutePath, 'index.d.ts');
} else if (!absolutePath.endsWith('.ts')) {
absolutePath += '.d.ts';
}
if (!ts.sys.fileExists(absolutePath)) return undefined;
return ts.sys.resolvePath(absolutePath);
}

function getEnclosingPackageJson(file: string): string | undefined {
let cache = state.packageTypings;
if (cache.has(file)) return cache.get(file);
let result = getEnclosingPackageJsonRaw(file);
cache.set(file, result);
return result;
}

function getEnclosingPackageJsonRaw(file: string): string | undefined {
let packageJson = pathlib.join(file, 'package.json');
if (ts.sys.fileExists(packageJson)) {
return packageJson;
}
if (pathlib.basename(file) === 'node_modules') {
return undefined;
}
let dirname = pathlib.dirname(file);
if (dirname.length < file.length) {
return getEnclosingPackageJson(dirname);
}
return undefined;
}

/**
* Debugging method for finding cycles in the TypeScript AST. Should not be used in production.
*
Expand Down Expand Up @@ -330,7 +242,6 @@ function extractFile(filename: string): string {
function prepareNextFile() {
if (state.pendingResponse != null) return;
if (state.pendingFileIndex < state.pendingFiles.length) {
checkMemoryUsage();
let nextFilename = state.pendingFiles[state.pendingFileIndex];
state.pendingResponse = extractFile(nextFilename);
}
Expand Down Expand Up @@ -368,13 +279,12 @@ function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolea
}

/**
* Gets the AST and source code for the given file, either from
* an already-open project, or by parsing the file.
* Gets the AST and source code by parsing the given file.
*/
function getAstForFile(filename: string): ts.SourceFile {
let { ast, code } = parseSingleFile(filename);
if (ast != null && isExtractableSourceFile(ast)) {
ast_extractor.augmentAst(ast, code, null);
ast_extractor.augmentAst(ast, code);
}
return ast;
}
Expand All @@ -401,7 +311,6 @@ function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string }
// parse `filename` with minimial checking
let compilerOptions: ts.CompilerOptions = {
experimentalDecorators: true,
experimentalAsyncFunctions: true,
jsx: ts.JsxEmit.Preserve,
noResolve: true,
};
Expand All @@ -413,98 +322,6 @@ function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string }
return { ast, code };
}

/**
* Matches a path segment referencing a package in a node_modules folder, and extracts
* two capture groups: the package name, and the relative path in the package.
*
* For example `lib/node_modules/@foo/bar/src/index.js` extracts the capture groups [`@foo/bar`, `src/index.js`].
*/
const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/;

interface LoadedConfig {
config: ts.ParsedCommandLine;
basePath: string;
packageEntryPoints: Map<string, string>;
packageJsonFiles: Map<string, string>;
virtualSourceRoot: VirtualSourceRoot;
ownFiles: string[];
}

function loadTsConfig(command: LoadCommand): LoadedConfig {
let tsConfig = ts.readConfigFile(command.tsConfig, ts.sys.readFile);
let basePath = pathlib.dirname(command.tsConfig);

let packageEntryPoints = new Map(command.packageEntryPoints);
let packageJsonFiles = new Map(command.packageJsonFiles);
let virtualSourceRoot = new VirtualSourceRoot(command.sourceRoot, command.virtualSourceRoot);

/**
* Rewrites path segments of form `node_modules/PACK/suffix` to be relative to
* the location of package PACK in the source tree, if it exists.
*/
function redirectNodeModulesPath(path: string) {
let nodeModulesMatch = nodeModulesRex.exec(path);
if (nodeModulesMatch == null) return null;
let packageName = nodeModulesMatch[1];
let packageJsonFile = packageJsonFiles.get(packageName);
if (packageJsonFile == null) return null;
let packageDir = pathlib.dirname(packageJsonFile);
let suffix = nodeModulesMatch[2];
let finalPath = pathlib.join(packageDir, suffix);
if (!ts.sys.fileExists(finalPath)) return null;
return finalPath;
}

/**
* Create the host passed to the tsconfig.json parser.
*
* We override its file system access in case there is an "extends"
* clause pointing into "./node_modules", which must be redirected to
* the location of an installed package or a checked-in package.
*/
let parseConfigHost: ts.ParseConfigHost = {
useCaseSensitiveFileNames: true,
readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
// Perform the glob matching in both real and virtual source roots.
let exclusions = excludes == null ? [] : [...excludes];
if (virtualSourceRoot.virtualSourceRoot != null) {
// qltest puts the virtual source root inside the real source root (.testproj).
// Make sure we don't find files inside the virtual source root in this pass.
exclusions.push(virtualSourceRoot.virtualSourceRoot);
}
let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
if (virtualDir == null) {
return originalResults;
}
// Make sure glob matching does not to discover anything in node_modules.
let virtualExclusions = excludes == null ? [] : [...excludes];
virtualExclusions.push('**/node_modules/**/*');
let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
return [...originalResults, ...virtualResults];
},
fileExists: (path: string) => {
return ts.sys.fileExists(path)
|| virtualSourceRoot.toVirtualPathIfFileExists(path) != null
|| redirectNodeModulesPath(path) != null;
},
readFile: (path: string) => {
if (!ts.sys.fileExists(path)) {
let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path);
if (virtualPath != null) return ts.sys.readFile(virtualPath);
virtualPath = redirectNodeModulesPath(path);
if (virtualPath != null) return ts.sys.readFile(virtualPath);
}
return ts.sys.readFile(path);
}
};
let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);

let ownFiles = config.fileNames.map(file => pathlib.resolve(file));

return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles };
}

function handleResetCommand(command: ResetCommand) {
reset();
console.log(JSON.stringify({
Expand Down Expand Up @@ -533,37 +350,6 @@ function reset() {
state = new State();
}

function getEnvironmentVariable<T>(name: string, parse: (x: string) => T, defaultValue: T) {
let value = process.env[name];
return value != null ? parse(value) : defaultValue;
}

/**
* Whether the memory usage was last observed to be above the threshold for restarting the TypeScript compiler.
*
* This is to prevent repeatedly restarting the compiler if the GC does not immediately bring us below the
* threshold again.
*/
let hasReloadedSinceExceedingThreshold = false;

/**
* If memory usage has moved above a the threshold, reboot the TypeScript compiler instance.
*
* Make sure to call this only when stdout has been flushed.
*/
function checkMemoryUsage() {
let bytesUsed = process.memoryUsage().heapUsed;
let megabytesUsed = bytesUsed / 1000000;
if (!hasReloadedSinceExceedingThreshold && megabytesUsed > reloadMemoryThresholdMb && state.project != null) {
console.warn('Restarting TypeScript compiler due to memory usage');
state.project.reload();
hasReloadedSinceExceedingThreshold = true;
}
else if (hasReloadedSinceExceedingThreshold && megabytesUsed < reloadMemoryThresholdMb) {
hasReloadedSinceExceedingThreshold = false;
}
}

function runReadLineInterface() {
reset();
let rl = readline.createInterface({ input: process.stdin, output: process.stdout });
Expand Down
Loading
Loading