From 7039d8bcbb636cd7d6644360efacc277953e7192 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 2 Oct 2026 16:07:06 -0400 Subject: [PATCH 1/2] Compute node bounds once at construction Body, Block, AnnotationExpression and the literal expressions computed their bounds lazily, so plugin edits could change them after the fact. Every node now sets its bounds in its constructor from what it was given. Empty blocks get theirs from the parent's constructor via Block.setBoundsBetween(), and the parser builds Body after its statements are known. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/plugins.md | 2 + src/parser/AstNode.spec.ts | 93 ++++++++++++++++++- src/parser/Expression.ts | 51 ++++------ src/parser/Parser.ts | 9 +- src/parser/Statement.ts | 186 +++++++++++++------------------------ 5 files changed, 179 insertions(+), 162 deletions(-) diff --git a/docs/plugins.md b/docs/plugins.md index ebb9a9d31..c15f923b4 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -588,6 +588,8 @@ To keep memory down, tokens and AST nodes don't store a `Location`. Instead, eac const location = util.getLocation(funcStmt.tokens.name); // { uri, range } ``` +A node's bounds are computed once, when it's constructed, from the children and tokens it's given. They describe where the node came from in the original source, so they don't change if a plugin edits the AST afterwards (i.e. adding statements to a block doesn't move the block). + `util.getLocation()` returns `undefined` for synthetic tokens and nodes (ones that were created in code rather than parsed from a file), since they have no position. A synthetic node won't get a source map entry, and diagnostics can't point at it. To give a synthetic token or node a position: diff --git a/src/parser/AstNode.spec.ts b/src/parser/AstNode.spec.ts index 87ccc4d93..6bc2f7143 100644 --- a/src/parser/AstNode.spec.ts +++ b/src/parser/AstNode.spec.ts @@ -7,12 +7,12 @@ import type { AAIndexedMemberExpression, AALiteralExpression, AAMemberExpression import { expectZeroDiagnostics } from '../testHelpers.spec'; import { tempDir, rootDir, outDir } from '../testHelpers.spec'; import { isAAIndexedMemberExpression, isAALiteralExpression, isAAMemberExpression, isAnnotationExpression, isArrayLiteralExpression, isAssignmentStatement, isBinaryExpression, isBlock, isCallExpression, isCallfuncExpression, isCatchStatement, isClassStatement, isConstStatement, isDimStatement, isDottedGetExpression, isDottedSetStatement, isEnumMemberStatement, isEnumStatement, isExpressionStatement, isForEachStatement, isForStatement, isFunctionExpression, isFunctionStatement, isGroupingExpression, isIfStatement, isIncrementStatement, isIndexedGetExpression, isIndexedSetStatement, isInterfaceFieldStatement, isInterfaceMethodStatement, isInterfaceStatement, isMethodStatement, isNamespaceStatement, isNewExpression, isNullCoalescingExpression, isPrintStatement, isReturnStatement, isTaggedTemplateStringExpression, isTemplateStringExpression, isTemplateStringQuasiExpression, isTernaryExpression, isThrowStatement, isTryCatchStatement, isTypecastExpression, isUnaryExpression, isVariableExpression, isLiteralExpression, isWhileStatement, isXmlAttributeGetExpression } from '../astUtils/reflection'; -import type { ClassStatement, FunctionStatement, InterfaceFieldStatement, InterfaceMethodStatement, MethodStatement, InterfaceStatement, CatchStatement, ThrowStatement, EnumStatement, EnumMemberStatement, ConstStatement, Block, PrintStatement, DimStatement, ForStatement, WhileStatement, IndexedSetStatement, NamespaceStatement, TryCatchStatement, DottedSetStatement, ExpressionStatement } from './Statement'; +import type { ConditionalCompileStatement, ClassStatement, FunctionStatement, InterfaceFieldStatement, InterfaceMethodStatement, MethodStatement, InterfaceStatement, CatchStatement, ThrowStatement, EnumStatement, EnumMemberStatement, ConstStatement, Block, PrintStatement, DimStatement, ForStatement, WhileStatement, IndexedSetStatement, NamespaceStatement, TryCatchStatement, DottedSetStatement, ExpressionStatement } from './Statement'; import { AssignmentStatement, EmptyStatement } from './Statement'; import { ParseMode, Parser } from './Parser'; import type { AstNode } from './AstNode'; import { WalkMode } from '../astUtils/visitors'; -import { isStatement } from '../astUtils/reflection'; +import { isConditionalCompileStatement, isStatement } from '../astUtils/reflection'; type DeepWriteable = { -readonly [P in keyof T]: DeepWriteable }; @@ -259,6 +259,95 @@ describe('AstNode', () => { }); }); + describe('bounds', () => { + it('keeps the bounds from construction when the AST is changed afterwards', () => { + const { ast } = Parser.parse('sub main()\n a = 1\nend sub'); + const block = ast.findChild(isBlock); + const bodyLocation = util.getLocation(ast); + const blockLocation = util.getLocation(block); + //a plugin moves a statement with a real position from elsewhere into this AST + const other = Parser.parse('\n\n\n\n\nsub other()\n b = 2\nend sub').ast; + ast.statements.push(other.statements[0]); + block.statements.push(other.findChild(isAssignmentStatement)); + expect(util.getLocation(ast)).to.eql(bodyLocation); + expect(util.getLocation(block)).to.eql(blockLocation); + }); + + it('computes the body bounds from its statements', () => { + const { ast } = Parser.parse('\n\nsub main()\nend sub\n\nsub other()\nend sub\n'); + expect(util.getLocation(ast).range).to.eql(util.createRange(2, 0, 6, 7)); + }); + + it('includes the annotation call in the annotation bounds', () => { + const { ast } = Parser.parse('@annotation(1, 2)\nsub main()\nend sub', { mode: ParseMode.BrighterScript }); + const annotation = (ast.statements[0] as FunctionStatement).annotations[0]; + expect(util.getLocation(annotation).range).to.eql(util.createRange(0, 0, 0, 17)); + }); + + describe('empty blocks', () => { + function getEmptyBlockRanges(code: string) { + const { ast } = Parser.parse(code, { mode: ParseMode.BrighterScript }); + return ast.findChildren(isBlock, { walkMode: WalkMode.visitAllRecursive }).map(x => util.getLocation(x)?.range); + } + + it('function', () => { + expect(getEmptyBlockRanges('sub main(a)\nend sub')).to.eql([ + util.createRange(0, 11, 1, 0) + ]); + }); + + it('if/else', () => { + expect(getEmptyBlockRanges('sub main()\n if true then\n else\n end if\nend sub')).to.eql([ + //the function body is not empty, so it's bounded by its statements + util.createRange(1, 4, 3, 10), + //then branch + util.createRange(1, 16, 2, 4), + //else branch + util.createRange(2, 8, 3, 4) + ]); + }); + + it('for', () => { + expect(getEmptyBlockRanges('sub main()\n for i = 0 to 1\n end for\nend sub')[1]).to.eql( + util.createRange(1, 18, 2, 4) + ); + }); + + it('for each', () => { + expect(getEmptyBlockRanges('sub main()\n for each x in y\n end for\nend sub')[1]).to.eql( + util.createRange(1, 19, 2, 4) + ); + }); + + it('while', () => { + expect(getEmptyBlockRanges('sub main()\n while true\n end while\nend sub')[1]).to.eql( + util.createRange(1, 14, 2, 4) + ); + }); + + it('try/catch', () => { + expect(getEmptyBlockRanges('sub main()\n try\n catch e\n end try\nend sub').slice(1)).to.eql([ + //try branch + util.createRange(1, 7, 2, 4), + //catch branch + util.createRange(2, 11, 3, 4) + ]); + }); + + it('#if/#else', () => { + //the walker skips inactive conditional compile branches, so read both branches off the statement directly + const { ast } = Parser.parse('#const DEBUG = true\n#if DEBUG\n#else\n#end if', { mode: ParseMode.BrighterScript }); + const statement = ast.findChild(isConditionalCompileStatement); + expect([statement.thenBranch, statement.elseBranch].map(x => util.getLocation(x)?.range)).to.eql([ + //then branch + util.createRange(1, 9, 2, 0), + //else branch + util.createRange(2, 5, 3, 0) + ]); + }); + }); + }); + describe('findChild', () => { it('finds a child that matches the matcher', () => { const file = program.setFile('source/main.brs', ` diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts index dc86087c1..be7a4a402 100644 --- a/src/parser/Expression.ts +++ b/src/parser/Expression.ts @@ -283,15 +283,16 @@ export class FunctionExpression extends Expression implements TypedefProvider { this.body.parent = this; } this.symbolTable = new SymbolTable('FunctionExpression', () => this.parent?.getSymbolTable()); - util.setBounds(this, + const signature = util.setBounds({} as Locatable, this.tokens.functionType, this.tokens.leftParen, ...this.parameters ?? [], this.tokens.rightParen, this.tokens.as, - this.returnTypeExpression, - this.tokens.endFunctionType + this.returnTypeExpression ); + util.setBounds(this, signature, this.tokens.endFunctionType); + this.body?.setBoundsBetween(signature, this.tokens.endFunctionType); } public readonly kind = AstNodeKind.FunctionExpression; @@ -1041,6 +1042,7 @@ export class LiteralExpression extends Expression { this.tokens = { value: options.value }; + util.setBounds(this, this.tokens.value); } public readonly tokens: { @@ -1049,15 +1051,9 @@ export class LiteralExpression extends Expression { public readonly kind = AstNodeKind.LiteralExpression; - public get pos() { - return this.tokens.value.pos; - } - public get end() { - return this.tokens.value.end; - } - public get source() { - return this.tokens.value.source; - } + public readonly pos: number; + public readonly end: number; + public readonly source: SourceInfo | undefined; public getType(options?: GetTypeOptions) { return util.tokenToBscType(this.tokens.value); @@ -2340,6 +2336,8 @@ export class AnnotationExpression extends Expression { }; this.call = options.call; this.name = this.tokens.name.text; + //the parser assigns `call` after construction (since the call needs this annotation as its callee), and updates the bounds then + util.setBounds(this, this.tokens.at, this.tokens.name, this.call); } public readonly kind = AstNodeKind.AnnotationExpression; @@ -2349,19 +2347,9 @@ export class AnnotationExpression extends Expression { readonly name: Token; }; - public get pos() { - return this.getBounds().pos; - } - public get end() { - return this.getBounds().end; - } - public get source() { - return this.getBounds().source; - } - private getBounds() { - //this needs to be computed on demand because `call` is assigned after construction - return util.setBounds({} as Locatable, this.tokens.at, this.tokens.name, this.call); - } + public readonly pos: number; + public readonly end: number; + public readonly source: SourceInfo | undefined; public readonly name: string; @@ -2668,6 +2656,7 @@ export class RegexLiteralExpression extends Expression { this.tokens = { regexLiteral: options.regexLiteral }; + util.setBounds(this, this.tokens.regexLiteral); } public readonly kind = AstNodeKind.RegexLiteralExpression; @@ -2675,15 +2664,9 @@ export class RegexLiteralExpression extends Expression { readonly regexLiteral: Token; }; - public get pos() { - return this.tokens?.regexLiteral?.pos; - } - public get end() { - return this.tokens?.regexLiteral?.end; - } - public get source() { - return this.tokens?.regexLiteral?.source; - } + public readonly pos: number; + public readonly end: number; + public readonly source: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { let text = this.tokens.regexLiteral?.text ?? ''; diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts index dc1d00dfe..12bdae9d3 100644 --- a/src/parser/Parser.ts +++ b/src/parser/Parser.ts @@ -244,7 +244,7 @@ export class Parser { private body() { const parentAnnotations = this.enterAnnotationBlock(); - let body = new Body({ statements: [] }); + const statements: Statement[] = []; if (this.tokens.length > 0) { this.consumeStatementSeparators(true); @@ -259,7 +259,7 @@ export class Parser { if (dec) { if (!isAnnotationExpression(dec)) { this.consumePendingAnnotations(dec); - body.statements.push(dec); + statements.push(dec); //ensure statement separator this.consumeStatementSeparators(false); } else { @@ -274,7 +274,8 @@ export class Parser { } this.exitAnnotationBlock(parentAnnotations); - return body; + //construct the body once all of its statements are known, so its bounds are computed from them + return new Body({ statements: statements }); } private sanitizeParseOptions(options: ParseOptions) { @@ -1752,6 +1753,8 @@ export class Parser { if (this.check(TokenKind.LeftParen)) { let leftParen = this.advance(); annotation.call = this.finishCall(leftParen, annotation, false); + //the call needs the annotation as its callee, so it can't be passed to the constructor. Include it in the bounds now that it exists + util.setBounds(annotation, annotation.tokens.at, annotation.tokens.name, annotation.call); } return annotation; } diff --git a/src/parser/Statement.ts b/src/parser/Statement.ts index f740e789e..64fe4cc92 100644 --- a/src/parser/Statement.ts +++ b/src/parser/Statement.ts @@ -10,7 +10,7 @@ import type { BrsTranspileState } from './BrsTranspileState'; import { ParseMode } from './Parser'; import type { WalkVisitor, WalkOptions } from '../astUtils/visitors'; import { InternalWalkMode, walk, createVisitor, WalkMode, walkArray } from '../astUtils/visitors'; -import { isBlock, isCallExpression, isCatchStatement, isClassType, isConditionalCompileStatement, isEnumMemberStatement, isEnumType, isEnumStatement, isExpressionStatement, isFieldStatement, isForEachStatement, isForStatement, isFunctionExpression, isFunctionStatement, isIfStatement, isInterfaceFieldStatement, isInterfaceMethodStatement, isInvalidType, isLiteralExpression, isMethodStatement, isNamespaceStatement, isPrintSeparatorExpression, isTryCatchStatement, isTypedefProvider, isUnaryExpression, isUninitializedType, isVoidType, isWhileStatement } from '../astUtils/reflection'; +import { isBlock, isCallExpression, isClassType, isConditionalCompileStatement, isEnumMemberStatement, isEnumType, isEnumStatement, isExpressionStatement, isFieldStatement, isFunctionStatement, isIfStatement, isInterfaceFieldStatement, isInterfaceMethodStatement, isInvalidType, isLiteralExpression, isMethodStatement, isNamespaceStatement, isPrintSeparatorExpression, isTypedefProvider, isUnaryExpression, isUninitializedType, isVoidType } from '../astUtils/reflection'; import type { GetTypeOptions } from '../interfaces'; import { TypeChainEntry, type TranspileResult, type TypedefProvider } from '../interfaces'; import { createDottedIdentifier, createIdentifier, createInvalidLiteral, createMethodStatement, createToken, createVariableExpression } from '../astUtils/creators'; @@ -67,6 +67,7 @@ export class Body extends Statement implements TypedefProvider { }) { super(); this.statements = options?.statements ?? []; + util.setBounds(this, ...this.statements); } public readonly statements: Statement[] = []; @@ -74,23 +75,9 @@ export class Body extends Statement implements TypedefProvider { public readonly symbolTable = new SymbolTable('Body', () => this.parent?.getSymbolTable()); - public get pos() { - return this.getBounds().pos; - } - public get end() { - return this.getBounds().end; - } - public get source() { - return this.getBounds().source; - } - private getBounds() { - //this needs to be lazy because the body has its statements pushed to it after being constructed - if (!this._bounds?.source) { - this._bounds = util.setBounds(this._bounds ?? {} as Locatable, ...(this.statements ?? [])); - } - return this._bounds; - } - private _bounds: Locatable; + public readonly pos: number; + public readonly end: number; + public readonly source: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result: TranspileResult = state.transpileAnnotations(this); @@ -318,122 +305,36 @@ export class Block extends Statement { super(); this.statements = options.statements; this.symbolTable = new SymbolTable('Block', () => this.parent.getSymbolTable()); + util.setBounds(this, ...this.statements ?? []); } public readonly statements: Statement[]; public readonly kind = AstNodeKind.Block; - private buildBounds(): Locatable { + public readonly pos: number; + public readonly end: number; + public readonly source: SourceInfo | undefined; + + /** + * An empty block has no statements to get its bounds from, so its parent calls this (from the parent's constructor) + * to set the bounds to the gap between the parent's code on either side of the block. Does nothing if the block has statements. + * @param before the parent's code just before this block (i.e. a function's signature) + * @param after the parent's code just after this block (i.e. `end function`) + */ + public setBoundsBetween(before: Locatable | undefined, after: Locatable | undefined) { if (this.statements?.length > 0) { - return util.setBounds({} as Locatable, ...this.statements ?? []); - } - let lastBitBefore: Locatable; - let firstBitAfter: Locatable; - - if (isFunctionExpression(this.parent)) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.tokens.functionType, - this.parent.tokens.leftParen, - ...(this.parent.parameters ?? []), - this.parent.tokens.rightParen, - this.parent.tokens.as, - this.parent.returnTypeExpression - ); - firstBitAfter = this.parent.tokens.endFunctionType; - } else if (isIfStatement(this.parent)) { - if (this.parent.thenBranch === this) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.tokens.then, - this.parent.condition - ); - firstBitAfter = util.setBounds({} as Locatable, - this.parent.tokens.else, - this.parent.elseBranch, - this.parent.tokens.endIf - ); - } else if (this.parent.elseBranch === this) { - lastBitBefore = this.parent.tokens.else; - firstBitAfter = this.parent.tokens.endIf; - } - } else if (isConditionalCompileStatement(this.parent)) { - if (this.parent.thenBranch === this) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.tokens.condition, - this.parent.tokens.not, - this.parent.tokens.hashIf - ); - firstBitAfter = util.setBounds({} as Locatable, - this.parent.tokens.hashElse, - this.parent.elseBranch, - this.parent.tokens.hashEndIf - ); - } else if (this.parent.elseBranch === this) { - lastBitBefore = this.parent.tokens.hashElse; - firstBitAfter = this.parent.tokens.hashEndIf; - } - } else if (isForStatement(this.parent)) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.increment, - this.parent.tokens.step, - this.parent.finalValue, - this.parent.tokens.to, - this.parent.counterDeclaration, - this.parent.tokens.for - ); - firstBitAfter = this.parent.tokens.endFor; - } else if (isForEachStatement(this.parent)) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.target, - this.parent.tokens.in, - this.parent.tokens.item, - this.parent.tokens.forEach - ); - firstBitAfter = this.parent.tokens.endFor; - } else if (isWhileStatement(this.parent)) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.condition, - this.parent.tokens.while - ); - firstBitAfter = this.parent.tokens.endWhile; - } else if (isTryCatchStatement(this.parent)) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.tokens.try - ); - firstBitAfter = util.setBounds({} as Locatable, - this.parent.tokens.endTry, - this.parent.catchStatement - ); - } else if (isCatchStatement(this.parent) && isTryCatchStatement(this.parent?.parent)) { - lastBitBefore = util.setBounds({} as Locatable, - this.parent.tokens.catch, - this.parent.exceptionVariableExpression - ); - firstBitAfter = this.parent.parent.tokens.endTry; + return; } - if (lastBitBefore?.source && firstBitAfter?.source) { - return { pos: lastBitBefore.end, end: firstBitAfter.pos, source: lastBitBefore.source }; + //offsets from different sources can't be compared, so both sides must come from the same source + if (before?.source && before.source === after?.source) { + const self = this as Locatable; + self.pos = util.getContentEnd(before); + self.end = after.pos; + self.source = before.source; } } - public get pos() { - return this.getBounds()?.pos; - } - public get end() { - return this.getBounds()?.end; - } - public get source() { - return this.getBounds()?.source; - } - private getBounds() { - //this needs to be lazy because the body has its statements pushed to it after being constructed - if (!this._bounds?.source) { - this._bounds = this.buildBounds(); - } - return this._bounds; - } - private _bounds: Locatable; - transpile(state: BrsTranspileState) { state.blockDepth++; let results = [] as TranspileResult; @@ -729,6 +630,15 @@ export class IfStatement extends Statement { endIf: options.endIf }; + //give empty branches the bounds of the gap they occupy (the else branch first, since the then branch ends where it starts) + if (isBlock(this.elseBranch)) { + this.elseBranch.setBoundsBetween(this.tokens.else, this.tokens.endIf); + } + this.thenBranch?.setBoundsBetween( + util.setBounds({} as Locatable, this.tokens.then, this.condition), + util.setBounds({} as Locatable, this.tokens.else, this.elseBranch, this.tokens.endIf) + ); + util.setBounds(this, ...Object.values(this.tokens), this.condition, @@ -1365,6 +1275,10 @@ export class ForStatement extends Statement { this.body = options.body; this.increment = options.increment; + this.body?.setBoundsBetween( + util.setBounds({} as Locatable, this.increment, this.tokens.step, this.finalValue, this.tokens.to, this.counterDeclaration, this.tokens.for), + this.tokens.endFor + ); util.setBounds(this, this.tokens.for, this.counterDeclaration, @@ -1497,6 +1411,10 @@ export class ForEachStatement extends Statement { this.target = options.target; this.typeExpression = options.typeExpression; + this.body?.setBoundsBetween( + util.setBounds({} as Locatable, this.target, this.tokens.in, this.typeExpression, this.tokens.as, this.tokens.item, this.tokens.forEach), + this.tokens.endFor + ); util.setBounds(this, this.tokens.forEach, this.tokens.item, @@ -1622,6 +1540,10 @@ export class WhileStatement extends Statement { }; this.body = options.body; this.condition = options.condition; + this.body?.setBoundsBetween( + util.setBounds({} as Locatable, this.condition, this.tokens.while), + this.tokens.endWhile + ); util.setBounds(this, this.tokens.while, this.condition, @@ -3808,6 +3730,15 @@ export class TryCatchStatement extends Statement { }; this.tryBranch = options.tryBranch; this.catchStatement = options.catchStatement; + this.tryBranch?.setBoundsBetween( + this.tokens.try, + util.setBounds({} as Locatable, this.tokens.endTry, this.catchStatement) + ); + //the catch statement doesn't know where the `end try` is, so the try/catch sets the bounds of its (empty) branch + this.catchStatement?.catchBranch?.setBoundsBetween( + util.setBounds({} as Locatable, this.catchStatement.tokens.catch, this.catchStatement.exceptionVariableExpression), + this.tokens.endTry + ); util.setBounds(this, this.tokens.try, this.tryBranch, @@ -4716,6 +4647,15 @@ export class ConditionalCompileStatement extends Statement { hashEndIf: options.hashEndIf }; + //give empty branches the bounds of the gap they occupy (the else branch first, since the then branch ends where it starts) + if (isBlock(this.elseBranch)) { + this.elseBranch.setBoundsBetween(this.tokens.hashElse, this.tokens.hashEndIf); + } + this.thenBranch?.setBoundsBetween( + util.setBounds({} as Locatable, this.tokens.condition, this.tokens.not, this.tokens.hashIf), + util.setBounds({} as Locatable, this.tokens.hashElse, this.elseBranch, this.tokens.hashEndIf) + ); + util.setBounds(this, ...Object.values(this.tokens), this.thenBranch, From 53424ae55c9162f008161bd035bc0475af66a783 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 15 Oct 2026 15:57:03 -0400 Subject: [PATCH 2/2] Rename Locatable props to match flatten-locations Co-Authored-By: Claude Opus 5.5 --- docs/plugins.md | 6 +- src/astUtils/creators.spec.ts | 4 +- src/astUtils/creators.ts | 12 +- src/astUtils/reflection.spec.ts | 6 +- src/bscPlugin/validation/BrsFileValidator.ts | 2 +- src/files/BrsFile.ts | 10 +- src/lexer/Lexer.ts | 12 +- src/lexer/Token.ts | 18 +- src/lexer/TokenObject.ts | 6 +- src/parser/AstNode.spec.ts | 2 +- src/parser/AstNode.ts | 18 +- src/parser/Expression.ts | 198 ++++++------- src/parser/Parser.ts | 16 +- src/parser/Statement.ts | 296 +++++++++---------- src/parser/TranspileState.spec.ts | 2 +- src/parser/TranspileState.ts | 18 +- src/parser/tests/Parser.spec.ts | 6 +- src/testHelpers.spec.ts | 4 +- src/util.spec.ts | 26 +- src/util.ts | 120 ++++---- 20 files changed, 393 insertions(+), 389 deletions(-) diff --git a/docs/plugins.md b/docs/plugins.md index 4bba616f2..a85d365bb 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -582,9 +582,9 @@ export default function () { ``` ## Locations -To keep memory down, tokens and AST nodes don't store a `Location`. Instead, each one stores where it starts and ends, counted from the start of the file (`pos` and `end`), plus a reference to info about the file it was parsed from (`source`), which is shared by everything from the same parse. +To keep memory down, tokens and AST nodes don't store a `Location`. Instead, each one stores where it starts and ends, counted from the start of the file (`startPosition` and `endPosition`), plus a reference to info about the file it was parsed from (`sourceInfo`), which is shared by everything from the same parse. `sourceInfo.linePositions` holds where each line of the file starts, which is how positions get converted to lines and characters. -Positions are the gaps between characters, like an editor cursor (and like an LSP `Position`). For a file that starts with `print`, the `print` token's `pos` is `0` (before the `p`) and its `end` is `5` (after the `t`), so `end - pos` is its length. +Positions are the gaps between characters, like an editor cursor (and like an LSP `Position`). For a file that starts with `print`, the `print` token's `startPosition` is `0` (before the `p`) and its `endPosition` is `5` (after the `t`), so `endPosition - startPosition` is its length. Build the line/character `Location` on demand with `util.getLocation()`: @@ -597,7 +597,7 @@ A node's bounds are computed once, when it's constructed, from the children and `util.getLocation()` returns `undefined` for synthetic tokens and nodes (ones that were created in code rather than parsed from a file), since they have no position. A synthetic node won't get a source map entry, and diagnostics can't point at it. A position describes where the code came from in the original source, so it's used as-is for source maps and diagnostics. Don't change the position of a parsed token or node. Only give one to a synthetic token or node you're creating: -- **Borrow it from a real token or node in the same file** (preferred): `util.setBounds(node, someToken)`. This shares the file's real `source`, so the node combines correctly with the rest of that file's AST. The `create*` functions do the same when given a token or node: `createToken(TokenKind.Identifier, 'name', someToken)`. +- **Borrow it from a real token or node in the same file** (preferred): `util.setBounds(node, someToken)`. This shares the file's real `sourceInfo`, so the node combines correctly with the rest of that file's AST. The `create*` functions do the same when given a token or node: `createToken(TokenKind.Identifier, 'name', someToken)`. - **From a `Location` in a file**: `util.setLocation(node, location, file.sourceInfo)`. This maps the location into that file, so the node combines with the rest of its AST. If the location is in a different file (or its lines aren't in the file), the node is left without a position. To pass one to a `create*` function, convert it first: `createToken(TokenKind.Identifier, 'name', util.setLocation({} as Locatable, location, file.sourceInfo))`. Code parsed from a snippet (i.e. `Parser.parse('...')` without a `srcPath`) has positions, but no uri. Those positions aren't in the file the code is added to, so lookups by position in that file (hover, definition, etc.) skip it, it isn't included in the bounds of nodes that also contain code from a file, and it doesn't get source map entries. diff --git a/src/astUtils/creators.spec.ts b/src/astUtils/creators.spec.ts index a01570996..a9ffb3296 100644 --- a/src/astUtils/creators.spec.ts +++ b/src/astUtils/creators.spec.ts @@ -27,12 +27,12 @@ describe('creators', () => { it('copies the bounds of a locatable', () => { const original = Lexer.scan('\n abc').tokens[1]; const token = createToken(TokenKind.Identifier, 'xyz', original); - expect(token.source).to.equal(original.source); + expect(token.sourceInfo).to.equal(original.sourceInfo); expect(util.getLocation(token)).to.eql(util.getLocation(original)); }); it('accepts a location from util.setLocation', () => { - const sourceInfo = Lexer.scan('\n\n abc', { srcPath: 'source/main.brs' }).tokens[0].source; + const sourceInfo = Lexer.scan('\n\n abc', { srcPath: 'source/main.brs' }).tokens[0].sourceInfo; const location = util.createLocation(2, 4, 2, 7, sourceInfo.uri); const locatable = util.setLocation({} as Locatable, location, sourceInfo); expect(util.getLocation(createToken(TokenKind.Identifier, 'xyz', locatable))).to.eql(location); diff --git a/src/astUtils/creators.ts b/src/astUtils/creators.ts index 44cf26942..ccd516e26 100644 --- a/src/astUtils/creators.ts +++ b/src/astUtils/creators.ts @@ -83,9 +83,9 @@ export function createToken(kind: T, text?: string, locatab kind, text ?? (tokenDefaults[kind as string] as string) ?? kind.toString().toLowerCase(), !text || text === kind.toString(), - locatable?.pos, - locatable?.end, - locatable?.source, + locatable?.startPosition, + locatable?.endPosition, + locatable?.sourceInfo, [] ) as Token as Token & { kind: T }; } @@ -95,9 +95,9 @@ export function createIdentifier(name: string, locatable?: Locatable): Identifie TokenKind.Identifier, name, false, - locatable?.pos, - locatable?.end, - locatable?.source, + locatable?.startPosition, + locatable?.endPosition, + locatable?.sourceInfo, [] ) as Identifier; } diff --git a/src/astUtils/reflection.spec.ts b/src/astUtils/reflection.spec.ts index af1569618..345c62177 100644 --- a/src/astUtils/reflection.spec.ts +++ b/src/astUtils/reflection.spec.ts @@ -190,9 +190,9 @@ describe('reflection', () => { kind: TokenKind.EscapedCharCodeLiteral, text: '0', isReserved: false, - pos: undefined, - end: undefined, - source: undefined, + startPosition: undefined, + endPosition: undefined, + sourceInfo: undefined, charCode: 0, leadingTrivia: [] }; diff --git a/src/bscPlugin/validation/BrsFileValidator.ts b/src/bscPlugin/validation/BrsFileValidator.ts index 13d4a16f7..d059d05b5 100644 --- a/src/bscPlugin/validation/BrsFileValidator.ts +++ b/src/bscPlugin/validation/BrsFileValidator.ts @@ -582,7 +582,7 @@ export class BrsFileValidator { const addVariable = (nameToken: Token | undefined, fallback: Locatable) => { const name = nameToken?.text?.toLowerCase(); if (name && !variableLocatables.has(name)) { - variableLocatables.set(name, nameToken.source ? nameToken : fallback); + variableLocatables.set(name, nameToken.sourceInfo ? nameToken : fallback); } }; // count all the parameters as they count as variables diff --git a/src/files/BrsFile.ts b/src/files/BrsFile.ts index 9af1affb9..ac029461f 100644 --- a/src/files/BrsFile.ts +++ b/src/files/BrsFile.ts @@ -305,14 +305,14 @@ export class BrsFile implements BscFile { public get sourceInfo(): SourceInfo | undefined { const tokens = this.parser?.tokens; //every token from the same parse shares the same source, and the last token (`Eof`) always exists - return tokens?.[tokens.length - 1]?.source; + return tokens?.[tokens.length - 1]?.sourceInfo; } /** * Does the token contain the given offset (inclusive on both ends, excluding any trailing newline) */ private tokenContainsOffset(token: Token, offset: number) { - return token.pos <= offset && offset <= util.getContentEnd(token); + return token.startPosition <= offset && offset <= util.getContentEnd(token); } /** @@ -339,7 +339,7 @@ export class BrsFile implements BscFile { return undefined; } for (let token of this.parser.tokens) { - if (offset < token.pos) { + if (offset < token.startPosition) { return token; } } @@ -362,7 +362,7 @@ export class BrsFile implements BscFile { this.ast.walk((node) => { const latestContainer = containingNode; //bsc walks depth-first - if (node.source === source && node.pos <= offset && offset <= node.end) { + if (node.sourceInfo === source && node.startPosition <= offset && offset <= node.endPosition) { containingNode = node; } //we had a match before, and don't now. this means we've finished walking down the whole way, and found our match @@ -897,7 +897,7 @@ export class BrsFile implements BscFile { return token; } //if the position less than this token range, then this position touches no token, - if (offset < token.pos) { + if (offset < token.startPosition) { let t = tokens[i - 1]; //return the token or the first token return t ? t : tokens[0]; diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 0da580cba..0a64cb938 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -83,7 +83,7 @@ export class Lexer { this.tokens = []; this.diagnostics = []; this.sourceInfo = this.options.trackLocations - ? { uri: util.pathToUri(options?.srcPath), lineStarts: [0] } + ? { uri: util.pathToUri(options?.srcPath), linePositions: [0] } : undefined; while (!this.isAtEnd()) { this.scanToken(); @@ -408,7 +408,7 @@ export class Lexer { this.addToken(TokenKind.Newline); this.start = this.current; - this.sourceInfo?.lineStarts.push(this.current); + this.sourceInfo?.linePositions.push(this.current); } /** @@ -524,7 +524,7 @@ export class Lexer { token.charCode = 10; //track the start of the next line - this.sourceInfo?.lineStarts.push(this.current); + this.sourceInfo?.linePositions.push(this.current); continue; } else if (this.check('\r') && this.peekNext() === '\n') { this.templateQuasiString(); @@ -538,7 +538,7 @@ export class Lexer { token.charCode = 10; //track the start of the next line - this.sourceInfo?.lineStarts.push(this.current); + this.sourceInfo?.linePositions.push(this.current); continue; //a lone `\r` is a newline too (the same as everywhere else in the lexer, and in LSP) @@ -550,7 +550,7 @@ export class Lexer { token.charCode = 13; //track the start of the next line - this.sourceInfo?.lineStarts.push(this.current); + this.sourceInfo?.linePositions.push(this.current); continue; //escaped chars @@ -1197,7 +1197,7 @@ export class Lexer { * @returns the location of `text` */ private locationOf(): Location { - return util.getLocation({ pos: this.start, end: this.current, source: this.sourceInfo }); + return util.getLocation({ startPosition: this.start, endPosition: this.current, sourceInfo: this.sourceInfo }); } } diff --git a/src/lexer/Token.ts b/src/lexer/Token.ts index 783ee3bec..def76d04e 100644 --- a/src/lexer/Token.ts +++ b/src/lexer/Token.ts @@ -30,18 +30,19 @@ export interface Locatable { /** * Where this item starts, counted from the start of the source. * - * Positions are the gaps between characters, like an editor cursor. For the source `print`, `pos` is `0` (before the `p`) and `end` is `5` (after the `t`). + * Positions are the gaps between characters, like an editor cursor. For the source `print`, `startPosition` is `0` (before the `p`) and `endPosition` is `5` (after the `t`). * Counted in UTF-16 code units, the same as an LSP `Position.character` */ - pos: number; + startPosition: number; /** - * Where this item ends: the gap just after its last character (see `pos`). `end - pos` is the item's length + * Where this item ends: the gap just after its last character, counted from the start of the source (see `startPosition`). + * `endPosition - startPosition` is the item's length */ - end: number; + endPosition: number; /** * Info about the source this item was parsed from. Shared by every item from the same parse. `undefined` for synthetic items */ - source: SourceInfo | undefined; + sourceInfo: SourceInfo | undefined; } /** @@ -53,9 +54,12 @@ export interface SourceInfo { */ uri: string; /** - * Where each line starts (the gap before its first character), counted the same way as `Locatable.pos` + * Where each line starts: the gap before its first character, counted from the start of the source in UTF-16 code units + * (the same as `Locatable.startPosition`). `linePositions[0]` is always `0`, and the index of each entry is its 0-based line number. + * + * For the source `a = 1\nb = 2`, this is `[0, 6]`. `util.getLocation()` uses it to convert a position into a line/character `Position` */ - lineStarts: number[]; + linePositions: number[]; } /** diff --git a/src/lexer/TokenObject.ts b/src/lexer/TokenObject.ts index e66b7a7eb..13b4757c1 100644 --- a/src/lexer/TokenObject.ts +++ b/src/lexer/TokenObject.ts @@ -12,9 +12,9 @@ export class TokenObject implements Token { public kind: TokenKind, public text: string, public isReserved: boolean, - public pos: number, - public end: number, - public source: SourceInfo | undefined, + public startPosition: number, + public endPosition: number, + public sourceInfo: SourceInfo | undefined, public leadingTrivia: Token[] | undefined ) { } diff --git a/src/parser/AstNode.spec.ts b/src/parser/AstNode.spec.ts index cd6840a38..ef994cb26 100644 --- a/src/parser/AstNode.spec.ts +++ b/src/parser/AstNode.spec.ts @@ -599,7 +599,7 @@ describe('AstNode', () => { //skip these properties if ( - ['parent', 'symbolTable', 'pos', 'end', 'source'].includes(key) || + ['parent', 'symbolTable', 'startPosition', 'endPosition', 'sourceInfo'].includes(key) || //this is a circular reference property or the `returnType` prop, skip it (isFunctionExpression(original) && (key === 'functionStatement' || key === 'returnType')) || //circular reference property for annotations diff --git a/src/parser/AstNode.ts b/src/parser/AstNode.ts index 15117c64a..f1de57ec1 100644 --- a/src/parser/AstNode.ts +++ b/src/parser/AstNode.ts @@ -19,18 +19,18 @@ import { isBlock, isBody, isFunctionParameterExpression } from '../astUtils/refl export abstract class AstNode implements Locatable { public abstract kind: AstNodeKind; /** - * Where this node starts, counted from the start of its source (see `Locatable.pos`). + * Where this node starts, counted from the start of its source (see `Locatable.startPosition`). * Use `util.getLocation()` to get the line/character `Location` */ - public abstract pos: number; + public abstract startPosition: number; /** - * Where this node ends: the gap just after its last character (see `Locatable.end`) + * Where this node ends: the gap just after its last character (see `Locatable.endPosition`) */ - public abstract end: number; + public abstract endPosition: number; /** * Info about the source this node was parsed from. `undefined` for synthetic nodes */ - public abstract source: SourceInfo | undefined; + public abstract sourceInfo: SourceInfo | undefined; /** * @deprecated use `util.getLocation(node)`. This is computed on every read @@ -170,13 +170,13 @@ export abstract class AstNode implements Locatable { //a position only means something within one source (the one this search starts from), so only nodes from that source can match. //Nodes without a source (i.e. a synthetic `if` wrapping real code) or from another source (i.e. code parsed from a snippet) never match //on their own, but the walk still steps into them, so the real code inside them can still be found - const searchSource = this.source; + const searchSource = this.sourceInfo; return this.findChild((node) => { - if (!node?.source || (searchSource && node.source !== searchSource)) { + if (!node?.sourceInfo || (searchSource && node.sourceInfo !== searchSource)) { return; } - const offset = util.getOffset(node.source, position); - if (offset !== undefined && node.pos <= offset && offset <= node.end) { + const offset = util.getOffset(node.sourceInfo, position); + if (offset !== undefined && node.startPosition <= offset && offset <= node.endPosition) { return node.findChildAtPosition(position, options) ?? node; } }, options); diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts index ea6ca57c0..6fbd10093 100644 --- a/src/parser/Expression.ts +++ b/src/parser/Expression.ts @@ -61,9 +61,9 @@ export class BinaryExpression extends Expression { public readonly kind = AstNodeKind.BinaryExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState): TranspileResult { return [ @@ -157,9 +157,9 @@ export class CallExpression extends Expression { public readonly kind = AstNodeKind.CallExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** `a.b()` -> `a.b` (not the args) */ public get previousInChain() { @@ -324,9 +324,9 @@ export class FunctionExpression extends Expression implements TypedefProvider { return this.tokens.endFunctionType?.leadingTrivia; } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState, name?: Identifier, includeBody = true) { let results = [] as TranspileResult; @@ -587,9 +587,9 @@ export class FunctionParameterExpression extends Expression { return paramType; } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState) { let result: TranspileResult = [ @@ -701,9 +701,9 @@ export class DottedGetExpression extends Expression { public readonly kind = AstNodeKind.DottedGetExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** `a.b` -> `a` */ public get previousInChain() { @@ -811,9 +811,9 @@ export class XmlAttributeGetExpression extends Expression { public readonly obj: Expression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** `a@b` -> `a` */ public get previousInChain() { @@ -914,9 +914,9 @@ export class IndexedGetExpression extends Expression { return this.indexes?.[0]; } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** `a[i]` -> `a` (not the index) */ public get previousInChain() { @@ -1003,9 +1003,9 @@ export class GroupingExpression extends Expression { public readonly kind = AstNodeKind.GroupingExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { if (isTypecastExpression(this.expression)) { @@ -1061,9 +1061,9 @@ export class LiteralExpression extends Expression { public readonly kind = AstNodeKind.LiteralExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public getType(options?: GetTypeOptions) { return util.tokenToBscType(this.tokens.value); @@ -1128,9 +1128,9 @@ export class PrintSeparatorExpression extends Expression { public readonly kind = AstNodeKind.PrintSeparatorExpression; - public pos: number; - public end: number; - public source: SourceInfo | undefined; + public startPosition: number; + public endPosition: number; + public sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -1174,9 +1174,9 @@ export class EscapedCharCodeLiteralExpression extends Expression { readonly value: Token & { charCode: number }; }; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -1221,9 +1221,9 @@ export class ArrayLiteralExpression extends Expression { public readonly kind = AstNodeKind.ArrayLiteralExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result: TranspileResult = []; @@ -1307,9 +1307,9 @@ export class AAMemberExpression extends Expression { public readonly kind = AstNodeKind.AAMemberExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly tokens: { readonly key: Token; @@ -1384,9 +1384,9 @@ export class AAIndexedMemberExpression extends Expression { public readonly kind = AstNodeKind.AAIndexedMemberExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return []; @@ -1436,9 +1436,9 @@ export class AALiteralExpression extends Expression { public readonly kind = AstNodeKind.AALiteralExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result: TranspileResult = []; @@ -1555,9 +1555,9 @@ export class UnaryExpression extends Expression { public readonly kind = AstNodeKind.UnaryExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly tokens: { readonly operator: Token; @@ -1623,9 +1623,9 @@ export class VariableExpression extends Expression { public readonly kind = AstNodeKind.VariableExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * @deprecated use `tokens.name` instead @@ -1717,9 +1717,9 @@ export class SourceLiteralExpression extends Expression { util.setBounds(this, this.tokens.value); } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly kind = AstNodeKind.SourceLiteralExpression; @@ -1867,9 +1867,9 @@ export class NewExpression extends Expression { public readonly kind = AstNodeKind.NewExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly tokens: { readonly new?: Token; @@ -1976,9 +1976,9 @@ export class CallfuncExpression extends Expression { public readonly kind = AstNodeKind.CallfuncExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** `a@.b()` -> `a` (not the args) */ public get previousInChain() { @@ -2064,9 +2064,9 @@ export class TemplateStringQuasiExpression extends Expression { public readonly expressions: Array; public readonly kind = AstNodeKind.TemplateStringQuasiExpression; - readonly pos: number; - readonly end: number; - readonly source: SourceInfo | undefined; + readonly startPosition: number; + readonly endPosition: number; + readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState, skipEmptyStrings = true) { let result = [] as TranspileResult; @@ -2133,9 +2133,9 @@ export class TemplateStringExpression extends Expression { public readonly quasis: TemplateStringQuasiExpression[]; public readonly expressions: Expression[]; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public getType(options: GetTypeOptions) { return StringType.instance; @@ -2259,9 +2259,9 @@ export class TaggedTemplateStringExpression extends Expression { public readonly quasis: TemplateStringQuasiExpression[]; public readonly expressions: Expression[]; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [] as TranspileResult; @@ -2357,9 +2357,9 @@ export class AnnotationExpression extends Expression { readonly name: Token; }; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly name: string; @@ -2434,9 +2434,9 @@ export class TernaryExpression extends Expression { public readonly kind = AstNodeKind.TernaryExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly tokens: { readonly questionMark?: Token; @@ -2559,9 +2559,9 @@ export class NullCoalescingExpression extends Expression { public readonly kind = AstNodeKind.NullCoalescingExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly tokens: { readonly questionQuestion?: Token; @@ -2674,9 +2674,9 @@ export class RegexLiteralExpression extends Expression { readonly regexLiteral: Token; }; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { let text = this.tokens.regexLiteral?.text ?? ''; @@ -2812,9 +2812,9 @@ export class TypeExpression extends Expression implements TypedefProvider { */ public readonly resolvedType?: BscType; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { //roku built-in names (rosgnode*, etc.) collapse to `dynamic` at transpile. @@ -2910,9 +2910,9 @@ export class TypecastExpression extends Expression { public typeExpression?: TypeExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { return this.obj.transpile(state); @@ -2976,9 +2976,9 @@ export class TypedArrayExpression extends Expression { public readonly kind = AstNodeKind.TypedArrayExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { //typed arrays (e.g. `float[]`, `float[][][]`) collapse to `dynamic` since @@ -3036,9 +3036,9 @@ export class InlineInterfaceExpression extends Expression { public readonly kind = AstNodeKind.InlineInterfaceExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; get leadingTrivia(): Token[] { return this.tokens.open?.leadingTrivia; @@ -3117,9 +3117,9 @@ export class InlineInterfaceMemberExpression extends Expression { public readonly typeExpression?: TypeExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; get leadingTrivia(): Token[] { return this.tokens.optional?.leadingTrivia ?? this.tokens.name.leadingTrivia; @@ -3197,9 +3197,9 @@ export class TypedFunctionTypeExpression extends Expression { public readonly params: FunctionParameterExpression[]; public readonly returnType?: TypeExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { return [this.getType({ flags: SymbolTypeFlag.typetime }).toTypeString()]; diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts index 7b82a038d..27a51fc42 100644 --- a/src/parser/Parser.ts +++ b/src/parser/Parser.ts @@ -922,9 +922,9 @@ export class Parser { TokenKind.Function, 'function', true, - this.peek().pos, - this.peek().end, - this.peek().source, + this.peek().startPosition, + this.peek().endPosition, + this.peek().sourceInfo, [] ); } @@ -1382,10 +1382,10 @@ export class Parser { const originalToken = exitToken; exitToken = createToken(TokenKind.Exit, exitText, originalToken); - exitToken.end = exitToken.pos + 4; + exitToken.endPosition = exitToken.startPosition + 4; this.tokens[this.current - 1] = exitToken; const newLoopToken = createToken(TokenKind.While, whileText, originalToken); - newLoopToken.pos += 4; + newLoopToken.startPosition += 4; this.tokens.splice(this.current, 0, newLoopToken); } @@ -1685,7 +1685,7 @@ export class Parser { this.diagnostics.push({ ...DiagnosticMessages.expectedIdentifier('typecast'), location: { - uri: typecastToken?.source?.uri, + uri: typecastToken?.sourceInfo?.uri, range: util.createBoundingRange(typecastToken, this.peek()) } }); @@ -1867,7 +1867,7 @@ export class Parser { this.diagnostics.push({ ...DiagnosticMessages.unterminatedTemplateExpression(), location: { - uri: openingBacktick?.source?.uri, + uri: openingBacktick?.sourceInfo?.uri, range: util.createBoundingRange(openingBacktick, this.peek()) } }); @@ -1886,7 +1886,7 @@ export class Parser { this.diagnostics.push({ ...DiagnosticMessages.unterminatedTemplateString(), location: { - uri: openingBacktick?.source?.uri, + uri: openingBacktick?.sourceInfo?.uri, range: util.createBoundingRange(openingBacktick, this.peek()) } }); diff --git a/src/parser/Statement.ts b/src/parser/Statement.ts index 3284759f1..ca37a9ec4 100644 --- a/src/parser/Statement.ts +++ b/src/parser/Statement.ts @@ -38,9 +38,9 @@ export class EmptyStatement extends Statement { /** * Create a negative range to indicate this is an interpolated location */ - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly kind = AstNodeKind.EmptyStatement; @@ -75,9 +75,9 @@ export class Body extends Statement implements TypedefProvider { public readonly symbolTable = new SymbolTable('Body', () => this.parent?.getSymbolTable()); - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result: TranspileResult = state.transpileAnnotations(this); @@ -173,9 +173,9 @@ export class AssignmentStatement extends Statement { public readonly kind = AstNodeKind.AssignmentStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -250,9 +250,9 @@ export class AugmentedAssignmentStatement extends Statement { public readonly kind = AstNodeKind.AugmentedAssignmentStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -312,9 +312,9 @@ export class Block extends Statement { public readonly kind = AstNodeKind.Block; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * An empty block has no statements to get its bounds from, so its parent calls this (from the parent's constructor) @@ -327,11 +327,11 @@ export class Block extends Statement { return; } //offsets from different sources can't be compared, so both sides must come from the same source - if (before?.source && before.source === after?.source) { + if (before?.sourceInfo && before.sourceInfo === after?.sourceInfo) { const self = this as Locatable; - self.pos = util.getContentEnd(before); - self.end = after.pos; - self.source = before.source; + self.startPosition = util.getContentEnd(before); + self.endPosition = after.startPosition; + self.sourceInfo = before.sourceInfo; } } @@ -412,9 +412,9 @@ export class ExpressionStatement extends Statement { public readonly expression: Expression; public readonly kind = AstNodeKind.ExpressionStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -472,9 +472,9 @@ export class ExitStatement extends Statement { public readonly kind = AstNodeKind.ExitStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -526,9 +526,9 @@ export class FunctionStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.FunctionStatement as AstNodeKind; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * Get the name of this expression based on the parse mode @@ -659,9 +659,9 @@ export class IfStatement extends Statement { public readonly kind = AstNodeKind.IfStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; get isInline() { const allLeadingTrivia = [ @@ -796,9 +796,9 @@ export class IncrementStatement extends Statement { public readonly kind = AstNodeKind.IncrementStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -884,9 +884,9 @@ export class PrintStatement extends Statement { return { valueCount: valueCount, commaCount: commaCount }; } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [ @@ -972,9 +972,9 @@ export class DimStatement extends Statement { public readonly kind = AstNodeKind.DimStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState) { let result: TranspileResult = [ @@ -1052,9 +1052,9 @@ export class GotoStatement extends Statement { public readonly kind = AstNodeKind.GotoStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -1103,9 +1103,9 @@ export class LabelStatement extends Statement { }; public readonly kind = AstNodeKind.LabelStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public get leadingTrivia(): Token[] { return this.tokens.name.leadingTrivia; @@ -1155,9 +1155,9 @@ export class ReturnStatement extends Statement { public readonly value?: Expression; public readonly kind = AstNodeKind.ReturnStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [] as TranspileResult; @@ -1207,9 +1207,9 @@ export class EndStatement extends Statement { }; public readonly kind = AstNodeKind.EndStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -1248,9 +1248,9 @@ export class StopStatement extends Statement { public readonly kind = AstNodeKind.StopStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -1328,9 +1328,9 @@ export class ForStatement extends Statement { public readonly kind = AstNodeKind.ForStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [] as TranspileResult; @@ -1463,9 +1463,9 @@ export class ForEachStatement extends Statement { public readonly kind = AstNodeKind.ForEachStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [] as TranspileResult; @@ -1584,9 +1584,9 @@ export class WhileStatement extends Statement { public readonly kind = AstNodeKind.WhileStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [] as TranspileResult; @@ -1677,9 +1677,9 @@ export class DottedSetStatement extends Statement { public readonly kind = AstNodeKind.DottedSetStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { //if the value is a compound assignment, don't add the obj, dot, name, or operator...the expression will handle that @@ -1773,9 +1773,9 @@ export class IndexedSetStatement extends Statement { public readonly kind = AstNodeKind.IndexedSetStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { const result = []; @@ -1855,9 +1855,9 @@ export class LibraryStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.LibraryStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let result = [] as TranspileResult; @@ -1931,9 +1931,9 @@ export class NamespaceStatement extends Statement implements TypedefProvider { return this.getName(ParseMode.BrighterScript); } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public getName(parseMode: ParseMode) { const sep = parseMode === ParseMode.BrighterScript ? '.' : '_'; @@ -2044,10 +2044,10 @@ export class ImportStatement extends Statement implements TypedefProvider { if (this.tokens.path) { //remove quotes this.filePath = this.tokens.path.text.replace(/"/g, ''); - if (this.tokens.path.source) { + if (this.tokens.path.sourceInfo) { //adjust the range to exclude the quotes - this.tokens.path.pos++; - this.tokens.path.end--; + this.tokens.path.startPosition++; + this.tokens.path.endPosition--; } } } @@ -2059,9 +2059,9 @@ export class ImportStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.ImportStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly filePath: string; @@ -2144,9 +2144,9 @@ export class InterfaceStatement extends Statement implements TypedefProvider { readonly endInterface?: Token; }; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public get fields(): InterfaceFieldStatement[] { return this.body.filter(x => isInterfaceFieldStatement(x)) as InterfaceFieldStatement[]; @@ -2352,9 +2352,9 @@ export class InterfaceFieldStatement extends Statement implements TypedefProvide public readonly typeExpression?: TypeExpression; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public readonly tokens: { readonly name: Identifier; @@ -2473,9 +2473,9 @@ export class InterfaceMethodStatement extends Statement implements TypedefProvid public readonly kind = AstNodeKind.InterfaceMethodStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * Get the name of this method. */ @@ -2703,9 +2703,9 @@ export class ClassStatement extends Statement implements TypedefProvider { public readonly methods = [] as MethodStatement[]; public readonly fields = [] as FieldStatement[]; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * Register all members (methods and fields) found in the given statements, @@ -3348,9 +3348,9 @@ export class MethodStatement extends FunctionStatement { return this.modifiers.find(x => accessModifiers.includes(x.kind)); } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * Get the name of this method. @@ -3449,9 +3449,9 @@ export class MethodStatement extends FunctionStatement { kind: TokenKind.Identifier, text: 'super', isReserved: false, - pos: state.classStatement.tokens.name.pos, - end: state.classStatement.tokens.name.end, - source: state.classStatement.tokens.name.source, + startPosition: state.classStatement.tokens.name.startPosition, + endPosition: state.classStatement.tokens.name.endPosition, + sourceInfo: state.classStatement.tokens.name.sourceInfo, leadingTrivia: [] } }), @@ -3459,18 +3459,18 @@ export class MethodStatement extends FunctionStatement { kind: TokenKind.LeftParen, text: '(', isReserved: false, - pos: state.classStatement.tokens.name.pos, - end: state.classStatement.tokens.name.end, - source: state.classStatement.tokens.name.source, + startPosition: state.classStatement.tokens.name.startPosition, + endPosition: state.classStatement.tokens.name.endPosition, + sourceInfo: state.classStatement.tokens.name.sourceInfo, leadingTrivia: [] }, closingParen: { kind: TokenKind.RightParen, text: ')', isReserved: false, - pos: state.classStatement.tokens.name.pos, - end: state.classStatement.tokens.name.end, - source: state.classStatement.tokens.name.source, + startPosition: state.classStatement.tokens.name.startPosition, + endPosition: state.classStatement.tokens.name.endPosition, + sourceInfo: state.classStatement.tokens.name.sourceInfo, leadingTrivia: [] }, args: [] @@ -3660,9 +3660,9 @@ export class FieldStatement extends Statement implements TypedefProvider { DynamicType.instance; } - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public get leadingTrivia(): Token[] { return this.tokens.accessModifier?.leadingTrivia ?? this.tokens.optional?.leadingTrivia ?? this.tokens.name.leadingTrivia; @@ -3780,9 +3780,9 @@ export class TryCatchStatement extends Statement { public readonly kind = AstNodeKind.TryCatchStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { return [ @@ -3854,9 +3854,9 @@ export class CatchStatement extends Statement { public readonly kind = AstNodeKind.CatchStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState): TranspileResult { return [ @@ -3917,9 +3917,9 @@ export class ThrowStatement extends Statement { public readonly kind = AstNodeKind.ThrowStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public transpile(state: BrsTranspileState) { const result = [ @@ -3994,9 +3994,9 @@ export class EnumStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.EnumStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public getMembers() { const result = [] as EnumMemberStatement[]; @@ -4194,9 +4194,9 @@ export class EnumMemberStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.EnumMemberStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; /** * The name of the member @@ -4297,9 +4297,9 @@ export class ConstStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.ConstStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; public get name() { return this.tokens.name.text; @@ -4404,9 +4404,9 @@ export class ContinueStatement extends Statement { public readonly kind = AstNodeKind.ContinueStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { //when targeting firmware without native `continue` support, rewrite into a jump to the @@ -4475,9 +4475,9 @@ export class TypecastStatement extends Statement { public readonly kind = AstNodeKind.TypecastStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { //the typecast statement is a comment just for debugging purposes @@ -4538,9 +4538,9 @@ export class ConditionalCompileErrorStatement extends Statement { public readonly kind = AstNodeKind.ConditionalCompileErrorStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -4604,9 +4604,9 @@ export class AliasStatement extends Statement { public readonly kind = AstNodeKind.AliasStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { //transpile to a comment just for debugging purposes @@ -4698,9 +4698,9 @@ export class ConditionalCompileStatement extends Statement { public readonly kind = AstNodeKind.ConditionalCompileStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { let results = [] as TranspileResult; @@ -4842,9 +4842,9 @@ export class ConditionalCompileConstStatement extends Statement { public readonly kind = AstNodeKind.ConditionalCompileConstStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { return [ @@ -4913,9 +4913,9 @@ export class TypeStatement extends Statement implements TypedefProvider { public readonly kind = AstNodeKind.TypeStatement; - public readonly pos: number; - public readonly end: number; - public readonly source: SourceInfo | undefined; + public readonly startPosition: number; + public readonly endPosition: number; + public readonly sourceInfo: SourceInfo | undefined; transpile(state: BrsTranspileState) { //type statements have no runtime representation, so they're stripped entirely diff --git a/src/parser/TranspileState.spec.ts b/src/parser/TranspileState.spec.ts index cfb09fc78..1e8362dc9 100644 --- a/src/parser/TranspileState.spec.ts +++ b/src/parser/TranspileState.spec.ts @@ -18,7 +18,7 @@ describe('TranspileState', () => { it('does not map code parsed from a snippet, since its position is not in any file', () => { const state = new TranspileState(srcPath, {}); const token = Lexer.scan('\n print').tokens[1]; - expect(token.source).to.exist; + expect(token.sourceInfo).to.exist; expect(state.tokenToSourceNode(token).line).to.be.null; expect(state.sourceNode(token, 'print').line).to.be.null; }); diff --git a/src/parser/TranspileState.ts b/src/parser/TranspileState.ts index a0f76bc5b..816260bf3 100644 --- a/src/parser/TranspileState.ts +++ b/src/parser/TranspileState.ts @@ -8,9 +8,9 @@ import { util } from '../util'; import type { TranspileResult } from '../interfaces'; interface TranspileToken { - pos?: number; - end?: number; - source?: SourceInfo; + startPosition?: number; + endPosition?: number; + sourceInfo?: SourceInfo; /** * Location for tokens that aren't `Locatable` (i.e. SGTokens) */ @@ -69,7 +69,7 @@ export class TranspileState { public newline = '\n'; private getSource(locatable: RangeLike) { - let srcPath = (locatable as Locatable)?.source?.uri ?? (locatable as { location: Location })?.location?.uri ?? (locatable as Location).uri; + let srcPath = (locatable as Locatable)?.sourceInfo?.uri ?? (locatable as { location: Location })?.location?.uri ?? (locatable as Location).uri; if (srcPath) { srcPath = util.uriToPath(srcPath); //if a sourceRoot is specified, use that instead of the rootDir @@ -90,7 +90,7 @@ export class TranspileState { */ public sourceNode(locatable: RangeLike, code: string | SourceNode | TranspileResult): SourceNode { //source maps only need the start, so skip building a whole `Location` for locatables - const start = (locatable && 'source' in locatable) + const start = (locatable && 'sourceInfo' in locatable) ? this.getLocatableStart(locatable) : util.extractRange(locatable)?.start; return util.sourceNodeFromTranspileResult( @@ -123,7 +123,7 @@ export class TranspileState { * Get the start position of a token. Source maps only need the start, so this skips building a whole `Location` */ private getTokenStart(token: TranspileToken): Position | undefined { - return token?.source + return token?.sourceInfo ? this.getLocatableStart(token as Locatable) : token?.location?.range?.start; } @@ -133,15 +133,15 @@ export class TranspileState { * that position belongs to. Mapping it would point at the same lines of whatever file is being transpiled, so leave it unmapped instead */ private getLocatableStart(locatable: Locatable): Position | undefined { - return locatable.source?.uri ? util.getStartPosition(locatable) : undefined; + return locatable.sourceInfo?.uri ? util.getStartPosition(locatable) : undefined; } /** * Does this token span more than one line? (a trailing newline doesn't count) */ private isMultiLine(token: TranspileToken, start: Position) { - if (token.source) { - const nextLineStart = token.source.lineStarts[start.line + 1]; + if (token.sourceInfo) { + const nextLineStart = token.sourceInfo.linePositions[start.line + 1]; return nextLineStart !== undefined && nextLineStart <= util.getContentEnd(token as Locatable); } return token.location?.range?.end?.line > start.line; diff --git a/src/parser/tests/Parser.spec.ts b/src/parser/tests/Parser.spec.ts index 5b8e9a759..000d45dc9 100644 --- a/src/parser/tests/Parser.spec.ts +++ b/src/parser/tests/Parser.spec.ts @@ -11,9 +11,9 @@ export function token(kind: TokenKind, text?: string): Token { kind: kind, text: text!, isReserved: ReservedWords.has((text ?? '').toLowerCase()), - pos: undefined, - end: undefined, - source: undefined, + startPosition: undefined, + endPosition: undefined, + sourceInfo: undefined, leadingTrivia: [] }; } diff --git a/src/testHelpers.spec.ts b/src/testHelpers.spec.ts index f4aa5676d..9bd6af53e 100644 --- a/src/testHelpers.spec.ts +++ b/src/testHelpers.spec.ts @@ -37,13 +37,13 @@ export const trim = undent; /** * A shared source for hand-built test tokens, where every line is 1000 characters long */ -const testSource: SourceInfo = { uri: undefined, lineStarts: Array.from({ length: 1000 }, (x, i) => i * 1000) }; +const testSource: SourceInfo = { uri: undefined, linePositions: Array.from({ length: 1000 }, (x, i) => i * 1000) }; /** * Create a `Locatable` for a hand-built test token from line/character positions */ export function testLocatable(startLine: number, startCharacter: number, endLine: number, endCharacter: number): Locatable { - return { pos: (startLine * 1000) + startCharacter, end: (endLine * 1000) + endCharacter, source: testSource }; + return { startPosition: (startLine * 1000) + startCharacter, endPosition: (endLine * 1000) + endCharacter, sourceInfo: testSource }; } const sinon = createSandbox(); diff --git a/src/util.spec.ts b/src/util.spec.ts index 04a361b3c..575e7ba6a 100644 --- a/src/util.spec.ts +++ b/src/util.spec.ts @@ -1135,8 +1135,8 @@ describe('util', () => { }); it('clears the bounds when nothing has a source', () => { - const result = util.setBounds({ pos: 1, end: 2, source: testLocatable(0, 0, 0, 0).source }, createToken(TokenKind.Identifier, 'a')); - expect(result).to.eql({ pos: undefined, end: undefined, source: undefined }); + const result = util.setBounds({ startPosition: 1, endPosition: 2, sourceInfo: testLocatable(0, 0, 0, 0).sourceInfo }, createToken(TokenKind.Identifier, 'a')); + expect(result).to.eql({ startPosition: undefined, endPosition: undefined, sourceInfo: undefined }); }); it('does not end at the start of the next line when the last locatable is a LF newline', () => { @@ -1185,7 +1185,7 @@ describe('util', () => { }); describe('getOffset', () => { - const source = Lexer.scan('ab\ncd\nef').tokens[0].source; + const source = Lexer.scan('ab\ncd\nef').tokens[0].sourceInfo; it('converts positions to offsets', () => { expect(util.getOffset(source, util.createPosition(0, 0))).to.eql(0); @@ -1209,37 +1209,37 @@ describe('util', () => { it('maps the location into the source, so it combines with that source', () => { const tokens = Lexer.scan('a = 1\nbb = 2', { srcPath: srcPath }).tokens; - const sourceInfo = tokens[0].source; + const sourceInfo = tokens[0].sourceInfo; const location = util.createLocation(1, 0, 1, 2, sourceInfo.uri); const result = util.setLocation({} as Locatable, location, sourceInfo); - expect(result.source).to.equal(sourceInfo); - expect(result.pos).to.eql(6); - expect(result.end).to.eql(8); + expect(result.sourceInfo).to.equal(sourceInfo); + expect(result.startPosition).to.eql(6); + expect(result.endPosition).to.eql(8); expect(util.getLocation(result)).to.eql(location); expect(util.getLocation(util.setBounds({} as Locatable, tokens[0], result)).range).to.eql(util.createRange(0, 0, 1, 2)); }); it('clears the position when given no location or source', () => { - const sourceInfo = Lexer.scan('a = 1', { srcPath: srcPath }).tokens[0].source; - expect(util.setLocation(testLocatable(0, 0, 0, 1), undefined, sourceInfo)).to.eql({ pos: undefined, end: undefined, source: undefined }); - expect(util.setLocation(testLocatable(0, 0, 0, 1), util.createLocation(0, 0, 0, 1), undefined)).to.eql({ pos: undefined, end: undefined, source: undefined }); + const sourceInfo = Lexer.scan('a = 1', { srcPath: srcPath }).tokens[0].sourceInfo; + expect(util.setLocation(testLocatable(0, 0, 0, 1), undefined, sourceInfo)).to.eql({ startPosition: undefined, endPosition: undefined, sourceInfo: undefined }); + expect(util.setLocation(testLocatable(0, 0, 0, 1), util.createLocation(0, 0, 0, 1), undefined)).to.eql({ startPosition: undefined, endPosition: undefined, sourceInfo: undefined }); }); it('clears the position for a location in a different file', () => { - const sourceInfo = Lexer.scan('a = 1', { srcPath: srcPath }).tokens[0].source; + const sourceInfo = Lexer.scan('a = 1', { srcPath: srcPath }).tokens[0].sourceInfo; const result = util.setLocation(testLocatable(0, 0, 0, 1), util.createLocation(0, 0, 0, 1, 'file:///other.brs'), sourceInfo); expect(util.getLocation(result)).to.be.undefined; }); it('clears the position for lines that are not in the source', () => { - const sourceInfo = Lexer.scan('a = 1', { srcPath: srcPath }).tokens[0].source; + const sourceInfo = Lexer.scan('a = 1', { srcPath: srcPath }).tokens[0].sourceInfo; //bsc builds ranges like this for comment flags that run to the end of the file const result = util.setLocation({} as Locatable, util.createLocation(0, 0, Number.MAX_SAFE_INTEGER, 0, sourceInfo.uri), sourceInfo); expect(util.getLocation(result)).to.be.undefined; }); it('keeps a character past the end of a line on that line', () => { - const sourceInfo = Lexer.scan('a = 1\nb = 2', { srcPath: srcPath }).tokens[0].source; + const sourceInfo = Lexer.scan('a = 1\nb = 2', { srcPath: srcPath }).tokens[0].sourceInfo; //bsc builds ranges like this for whole-file diagnostics const result = util.setLocation({} as Locatable, util.createLocation(0, 0, 0, Number.MAX_VALUE, sourceInfo.uri), sourceInfo); expect(util.getLocation(result).range.end.line).to.eql(0); diff --git a/src/util.ts b/src/util.ts index 24b8b50e3..aaca7f74b 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1070,7 +1070,7 @@ export class Util { public extractRange(rangeIsh: RangeLike): Range | undefined { if (!rangeIsh) { return undefined; - } else if ('source' in rangeIsh) { + } else if ('sourceInfo' in rangeIsh) { return this.getLocation(rangeIsh)?.range; } else if ('location' in rangeIsh) { return rangeIsh.location?.range; @@ -1268,9 +1268,9 @@ export class Util { token.kind, token.text, token.isReserved, - token.pos, - token.end, - token.source, + token.startPosition, + token.endPosition, + token.sourceInfo, token.leadingTrivia ? token.leadingTrivia.map(x => this.cloneToken(x)) : undefined ) as Token; //handle those tokens that have charCode @@ -1296,7 +1296,7 @@ export class Util { let range: Range; if (!locatable) { continue; - } else if ('source' in locatable) { + } else if ('sourceInfo' in locatable) { const location = this.getLocation(locatable); range = location?.range; if (!uri) { @@ -1354,30 +1354,30 @@ export class Util { } /** - * Set the `pos`, `end`, and `source` of `target` to the bounds of all the locatables. - * Offsets from different sources can't be compared, so only locatables from one source are included: the first one that has a uri, + * Set the `startPosition`, `endPosition`, and `sourceInfo` of `target` to the bounds of all the locatables. + * Positions from different sources can't be compared, so only locatables from one source are included: the first one that has a uri, * or the first one if none of them do. That way, code parsed from a snippet (which has no uri) never takes over the bounds of real code from a file. * A trailing newline is not included in the bounds, so the target never ends at the start of the next line */ public setBounds(target: T, ...locatables: Array): T { - let pos: number; - let end: number; - let source: SourceInfo; + let startPosition: number; + let endPosition: number; + let sourceInfo: SourceInfo; for (const locatable of locatables) { - if (!locatable?.source) { + if (!locatable?.sourceInfo) { continue; - } else if (!source || (!source.uri && locatable.source.uri)) { - source = locatable.source; - pos = locatable.pos; - end = this.getContentEnd(locatable); - } else if (locatable.source === source) { - pos = Math.min(pos, locatable.pos); - end = Math.max(end, this.getContentEnd(locatable)); - } - } - target.pos = pos; - target.end = end; - target.source = source; + } else if (!sourceInfo || (!sourceInfo.uri && locatable.sourceInfo.uri)) { + sourceInfo = locatable.sourceInfo; + startPosition = locatable.startPosition; + endPosition = this.getContentEnd(locatable); + } else if (locatable.sourceInfo === sourceInfo) { + startPosition = Math.min(startPosition, locatable.startPosition); + endPosition = Math.max(endPosition, this.getContentEnd(locatable)); + } + } + target.startPosition = startPosition; + target.endPosition = endPosition; + target.sourceInfo = sourceInfo; return target; } @@ -1391,22 +1391,22 @@ export class Util { */ public setLocation(target: T, location: Location | undefined, sourceInfo: SourceInfo | undefined): T { const range = location?.range; - const pos = this.getOffset(sourceInfo, range?.start); - const end = this.getOffset(sourceInfo, range?.end); - if (pos === undefined || end === undefined || (location.uri && location.uri !== sourceInfo.uri)) { - target.source = undefined; - target.pos = undefined; - target.end = undefined; + const startPosition = this.getOffset(sourceInfo, range?.start); + const endPosition = this.getOffset(sourceInfo, range?.end); + if (startPosition === undefined || endPosition === undefined || (location.uri && location.uri !== sourceInfo.uri)) { + target.sourceInfo = undefined; + target.startPosition = undefined; + target.endPosition = undefined; } else { - target.source = sourceInfo; - target.pos = pos; - target.end = end; + target.sourceInfo = sourceInfo; + target.startPosition = startPosition; + target.endPosition = endPosition; } return target; } /** - * Get the end offset of a locatable, excluding any trailing newline (i.e. a `Newline` token, or an escaped newline in a template string) + * Get the end position of a locatable, excluding any trailing newline (i.e. a `Newline` token, or an escaped newline in a template string) */ public getContentEnd(locatable: Locatable): number { const text = (locatable as Token).text; @@ -1414,73 +1414,73 @@ export class Util { const lastCharCode = text.charCodeAt(text.length - 1); // \n (and \r\n) if (lastCharCode === 10) { - return locatable.end - (text.length > 1 && text.charCodeAt(text.length - 2) === 13 ? 2 : 1); + return locatable.endPosition - (text.length > 1 && text.charCodeAt(text.length - 2) === 13 ? 2 : 1); // \r } else if (lastCharCode === 13) { - return locatable.end - 1; + return locatable.endPosition - 1; } } - return locatable.end; + return locatable.endPosition; } /** - * Build a line/character `Location` for the locatable. Returns `undefined` for synthetic items (those without a `source`) + * Build a line/character `Location` for the locatable. Returns `undefined` for synthetic items (those without a `sourceInfo`) */ public getLocation(locatable: Locatable): Location | undefined { - const source = locatable?.source; - if (!source) { + const sourceInfo = locatable?.sourceInfo; + if (!sourceInfo) { return undefined; } - const lineStarts = source.lineStarts; - const startLine = this.getLineIndex(lineStarts, locatable.pos, 0); - let endLine = this.getLineIndex(lineStarts, locatable.end, startLine); + const linePositions = sourceInfo.linePositions; + const startLine = this.getLineIndex(linePositions, locatable.startPosition, 0); + let endLine = this.getLineIndex(linePositions, locatable.endPosition, startLine); //a token that ends with a newline ends on its own line rather than at the start of the next one - if (endLine > startLine && lineStarts[endLine] === locatable.end && this.getContentEnd(locatable) !== locatable.end) { + if (endLine > startLine && linePositions[endLine] === locatable.endPosition && this.getContentEnd(locatable) !== locatable.endPosition) { endLine--; } return { - uri: source.uri, - range: this.createRange(startLine, locatable.pos - lineStarts[startLine], endLine, locatable.end - lineStarts[endLine]) + uri: sourceInfo.uri, + range: this.createRange(startLine, locatable.startPosition - linePositions[startLine], endLine, locatable.endPosition - linePositions[endLine]) }; } /** - * Get just the start `Position` of a locatable. Cheaper than `getLocation()` when the end isn't needed. - * Returns `undefined` for synthetic items (those without a `source`) + * Get just the start line/character `Position` of a locatable. Cheaper than `getLocation()` when the end isn't needed. + * Returns `undefined` for synthetic items (those without a `sourceInfo`) */ public getStartPosition(locatable: Locatable): Position | undefined { - const lineStarts = locatable?.source?.lineStarts; - if (!lineStarts) { + const linePositions = locatable?.sourceInfo?.linePositions; + if (!linePositions) { return undefined; } - const line = this.getLineIndex(lineStarts, locatable.pos, 0); - return this.createPosition(line, locatable.pos - lineStarts[line]); + const line = this.getLineIndex(linePositions, locatable.startPosition, 0); + return this.createPosition(line, locatable.startPosition - linePositions[line]); } /** - * Convert a line/character `Position` into an absolute offset within the given source. + * Convert a line/character `Position` into a position counted from the start of the source (see `Locatable.startPosition`). * A character past the end of its line is clamped to that line, so it never lands on the next line. * Returns `undefined` if the source or the position's line is missing */ - public getOffset(source: SourceInfo | undefined, position: Position | undefined): number | undefined { - const lineStarts = source?.lineStarts; - const lineStart = lineStarts?.[position?.line]; + public getOffset(sourceInfo: SourceInfo | undefined, position: Position | undefined): number | undefined { + const linePositions = sourceInfo?.linePositions; + const lineStart = linePositions?.[position?.line]; if (lineStart === undefined) { return undefined; } const offset = lineStart + position.character; - const nextLineStart = lineStarts[position.line + 1]; + const nextLineStart = linePositions[position.line + 1]; return nextLineStart !== undefined && offset >= nextLineStart ? nextLineStart - 1 : offset; } /** - * Binary search `lineStarts` (beginning at line index `low`) for the index of the line that contains `offset` + * Binary search `linePositions` (beginning at line index `low`) for the index of the line that contains `position` */ - private getLineIndex(lineStarts: number[], offset: number, low: number) { - let high = lineStarts.length - 1; + private getLineIndex(linePositions: number[], position: number, low: number) { + let high = linePositions.length - 1; while (low < high) { const mid = Math.floor((low + high + 1) / 2); - if (lineStarts[mid] <= offset) { + if (linePositions[mid] <= position) { low = mid; } else { high = mid - 1;