Rename Locatable props to startPosition, endPosition and sourceInfo

docs/plugins.md

@@ -582,9 +582,9 @@ export default function () {
 ```
 
 ## Locations
-To keep memory down, tokens and AST nodes don't store a `Location`. Instead, each one stores where it starts and ends, counted from the start of the file (`pos` and `end`), plus a reference to info about the file it was parsed from (`source`), which is shared by everything from the same parse.
+To keep memory down, tokens and AST nodes don't store a `Location`. Instead, each one stores where it starts and ends, counted from the start of the file (`startPosition` and `endPosition`), plus a reference to info about the file it was parsed from (`sourceInfo`), which is shared by everything from the same parse. `sourceInfo.linePositions` holds where each line of the file starts, which is how positions get converted to lines and characters.
 
-Positions are the gaps between characters, like an editor cursor (and like an LSP `Position`). For a file that starts with `print`, the `print` token's `pos` is `0` (before the `p`) and its `end` is `5` (after the `t`), so `end - pos` is its length.
+Positions are the gaps between characters, like an editor cursor (and like an LSP `Position`). For a file that starts with `print`, the `print` token's `startPosition` is `0` (before the `p`) and its `endPosition` is `5` (after the `t`), so `endPosition - startPosition` is its length.
 
 Build the line/character `Location` on demand with `util.getLocation()`:
 
@@ -595,7 +595,7 @@ const location = util.getLocation(funcStmt.tokens.name); // { uri, range }
 `util.getLocation()` returns `undefined` for synthetic tokens and nodes (ones that were created in code rather than parsed from a file), since they have no position. A synthetic node won't get a source map entry, and diagnostics can't point at it.
 
 A position describes where the code came from in the original source, so it's used as-is for source maps and diagnostics. Don't change the position of a parsed token or node. Only give one to a synthetic token or node you're creating:
-- **Borrow it from a real token or node in the same file** (preferred): `util.setBounds(node, someToken)`. This shares the file's real `source`, so the node combines correctly with the rest of that file's AST. The `create*` functions do the same when given a token or node: `createToken(TokenKind.Identifier, 'name', someToken)`.
+- **Borrow it from a real token or node in the same file** (preferred): `util.setBounds(node, someToken)`. This shares the file's real `sourceInfo`, so the node combines correctly with the rest of that file's AST. The `create*` functions do the same when given a token or node: `createToken(TokenKind.Identifier, 'name', someToken)`.
 - **From a `Location` in a file**: `util.setLocation(node, location, file.sourceInfo)`. This maps the location into that file, so the node combines with the rest of its AST. If the location is in a different file (or its lines aren't in the file), the node is left without a position. To pass one to a `create*` function, convert it first: `createToken(TokenKind.Identifier, 'name', util.setLocation({} as Locatable, location, file.sourceInfo))`.
 
 Code parsed from a snippet (i.e. `Parser.parse('...')` without a `srcPath`) has positions, but no uri. Those positions aren't in the file the code is added to, so lookups by position in that file (hover, definition, etc.) skip it, it isn't included in the bounds of nodes that also contain code from a file, and it doesn't get source map entries.

src/astUtils/creators.spec.ts

@@ -27,12 +27,12 @@ describe('creators', () => {
         it('copies the bounds of a locatable', () => {
             const original = Lexer.scan('\n  abc').tokens[1];
             const token = createToken(TokenKind.Identifier, 'xyz', original);
-            expect(token.source).to.equal(original.source);
+            expect(token.sourceInfo).to.equal(original.sourceInfo);
             expect(util.getLocation(token)).to.eql(util.getLocation(original));
         });
 
         it('accepts a location from util.setLocation', () => {
-            const sourceInfo = Lexer.scan('\n\n    abc', { srcPath: 'source/main.brs' }).tokens[0].source;
+            const sourceInfo = Lexer.scan('\n\n    abc', { srcPath: 'source/main.brs' }).tokens[0].sourceInfo;
             const location = util.createLocation(2, 4, 2, 7, sourceInfo.uri);
             const locatable = util.setLocation({} as Locatable, location, sourceInfo);
             expect(util.getLocation(createToken(TokenKind.Identifier, 'xyz', locatable))).to.eql(location);

src/astUtils/creators.ts

@@ -83,9 +83,9 @@ export function createToken<T extends TokenKind>(kind: T, text?: string, locatab
         kind,
         text ?? (tokenDefaults[kind as string] as string) ?? kind.toString().toLowerCase(),
         !text || text === kind.toString(),
-        locatable?.pos,
-        locatable?.end,
-        locatable?.source,
+        locatable?.startPosition,
+        locatable?.endPosition,
+        locatable?.sourceInfo,
         []
     ) as Token as Token & { kind: T };
 }
@@ -95,9 +95,9 @@ export function createIdentifier(name: string, locatable?: Locatable): Identifie
         TokenKind.Identifier,
         name,
         false,
-        locatable?.pos,
-        locatable?.end,
-        locatable?.source,
+        locatable?.startPosition,
+        locatable?.endPosition,
+        locatable?.sourceInfo,
         []
     ) as Identifier;
 }

src/astUtils/reflection.spec.ts

@@ -190,9 +190,9 @@ describe('reflection', () => {
             kind: TokenKind.EscapedCharCodeLiteral,
             text: '0',
             isReserved: false,
-            pos: undefined,
-            end: undefined,
-            source: undefined,
+            startPosition: undefined,
+            endPosition: undefined,
+            sourceInfo: undefined,
             charCode: 0,
             leadingTrivia: []
         };

src/bscPlugin/validation/BrsFileValidator.ts

@@ -582,7 +582,7 @@ export class BrsFileValidator {
         const addVariable = (nameToken: Token | undefined, fallback: Locatable) => {
             const name = nameToken?.text?.toLowerCase();
             if (name && !variableLocatables.has(name)) {
-                variableLocatables.set(name, nameToken.source ? nameToken : fallback);
+                variableLocatables.set(name, nameToken.sourceInfo ? nameToken : fallback);
             }
         };
         // count all the parameters as they count as variables

src/files/BrsFile.ts

@@ -305,14 +305,14 @@ export class BrsFile implements BscFile {
     public get sourceInfo(): SourceInfo | undefined {
         const tokens = this.parser?.tokens;
         //every token from the same parse shares the same source, and the last token (`Eof`) always exists
-        return tokens?.[tokens.length - 1]?.source;
+        return tokens?.[tokens.length - 1]?.sourceInfo;
     }
 
     /**
      * Does the token contain the given offset (inclusive on both ends, excluding any trailing newline)
      */
     private tokenContainsOffset(token: Token, offset: number) {
-        return token.pos <= offset && offset <= util.getContentEnd(token);
+        return token.startPosition <= offset && offset <= util.getContentEnd(token);
     }
 
     /**
@@ -339,7 +339,7 @@ export class BrsFile implements BscFile {
             return undefined;
         }
         for (let token of this.parser.tokens) {
-            if (offset < token.pos) {
+            if (offset < token.startPosition) {
                 return token;
             }
         }
@@ -362,7 +362,7 @@ export class BrsFile implements BscFile {
         this.ast.walk((node) => {
             const latestContainer = containingNode;
             //bsc walks depth-first
-            if (node.source === source && node.pos <= offset && offset <= node.end) {
+            if (node.sourceInfo === source && node.startPosition <= offset && offset <= node.endPosition) {
                 containingNode = node;
             }
             //we had a match before, and don't now. this means we've finished walking down the whole way, and found our match
@@ -897,7 +897,7 @@ export class BrsFile implements BscFile {
                 return token;
             }
             //if the position less than this token range, then this position touches no token,
-            if (offset < token.pos) {
+            if (offset < token.startPosition) {
                 let t = tokens[i - 1];
                 //return the token or the first token
                 return t ? t : tokens[0];

src/lexer/Lexer.ts

@@ -83,7 +83,7 @@ export class Lexer {
         this.tokens = [];
         this.diagnostics = [];
         this.sourceInfo = this.options.trackLocations
-            ? { uri: util.pathToUri(options?.srcPath), lineStarts: [0] }
+            ? { uri: util.pathToUri(options?.srcPath), linePositions: [0] }
             : undefined;
         while (!this.isAtEnd()) {
             this.scanToken();
@@ -408,7 +408,7 @@ export class Lexer {
 
         this.addToken(TokenKind.Newline);
         this.start = this.current;
-        this.sourceInfo?.lineStarts.push(this.current);
+        this.sourceInfo?.linePositions.push(this.current);
     }
 
     /**
@@ -524,7 +524,7 @@ export class Lexer {
                 token.charCode = 10;
 
                 //track the start of the next line
-                this.sourceInfo?.lineStarts.push(this.current);
+                this.sourceInfo?.linePositions.push(this.current);
                 continue;
             } else if (this.check('\r') && this.peekNext() === '\n') {
                 this.templateQuasiString();
@@ -538,7 +538,7 @@ export class Lexer {
                 token.charCode = 10;
 
                 //track the start of the next line
-                this.sourceInfo?.lineStarts.push(this.current);
+                this.sourceInfo?.linePositions.push(this.current);
                 continue;
 
                 //a lone `\r` is a newline too (the same as everywhere else in the lexer, and in LSP)
@@ -550,7 +550,7 @@ export class Lexer {
                 token.charCode = 13;
 
                 //track the start of the next line
-                this.sourceInfo?.lineStarts.push(this.current);
+                this.sourceInfo?.linePositions.push(this.current);
                 continue;
 
                 //escaped chars
@@ -1197,7 +1197,7 @@ export class Lexer {
      * @returns the location of `text`
      */
     private locationOf(): Location {
-        return util.getLocation({ pos: this.start, end: this.current, source: this.sourceInfo });
+        return util.getLocation({ startPosition: this.start, endPosition: this.current, sourceInfo: this.sourceInfo });
     }
 }
 

src/lexer/Token.ts

@@ -30,18 +30,19 @@ export interface Locatable {
     /**
      * Where this item starts, counted from the start of the source.
      *
-     * Positions are the gaps between characters, like an editor cursor. For the source `print`, `pos` is `0` (before the `p`) and `end` is `5` (after the `t`).
+     * Positions are the gaps between characters, like an editor cursor. For the source `print`, `startPosition` is `0` (before the `p`) and `endPosition` is `5` (after the `t`).
      * Counted in UTF-16 code units, the same as an LSP `Position.character`
      */
-    pos: number;
+    startPosition: number;
     /**
-     * Where this item ends: the gap just after its last character (see `pos`). `end - pos` is the item's length
+     * Where this item ends: the gap just after its last character, counted from the start of the source (see `startPosition`).
+     * `endPosition - startPosition` is the item's length
      */
-    end: number;
+    endPosition: number;
     /**
      * Info about the source this item was parsed from. Shared by every item from the same parse. `undefined` for synthetic items
      */
-    source: SourceInfo | undefined;
+    sourceInfo: SourceInfo | undefined;
 }
 
 /**
@@ -53,9 +54,12 @@ export interface SourceInfo {
      */
     uri: string;
     /**
-     * Where each line starts (the gap before its first character), counted the same way as `Locatable.pos`
+     * Where each line starts: the gap before its first character, counted from the start of the source in UTF-16 code units
+     * (the same as `Locatable.startPosition`). `linePositions[0]` is always `0`, and the index of each entry is its 0-based line number.
+     *
+     * For the source `a = 1\nb = 2`, this is `[0, 6]`. `util.getLocation()` uses it to convert a position into a line/character `Position`
      */
-    lineStarts: number[];
+    linePositions: number[];
 }
 
 /**

src/lexer/TokenObject.ts

@@ -12,9 +12,9 @@ export class TokenObject implements Token {
         public kind: TokenKind,
         public text: string,
         public isReserved: boolean,
-        public pos: number,
-        public end: number,
-        public source: SourceInfo | undefined,
+        public startPosition: number,
+        public endPosition: number,
+        public sourceInfo: SourceInfo | undefined,
         public leadingTrivia: Token[] | undefined
     ) { }
 

src/parser/AstNode.spec.ts

@@ -510,7 +510,7 @@ describe('AstNode', () => {
 
                     //skip these properties
                     if (
-                        ['parent', 'symbolTable', 'pos', 'end', 'source'].includes(key) ||
+                        ['parent', 'symbolTable', 'startPosition', 'endPosition', 'sourceInfo'].includes(key) ||
                         //this is a circular reference property or the `returnType` prop, skip it
                         (isFunctionExpression(original) && (key === 'functionStatement' || key === 'returnType')) ||
                         //circular reference property for annotations

src/parser/AstNode.ts

@@ -19,18 +19,18 @@ import { isBlock, isBody, isFunctionParameterExpression } from '../astUtils/refl
 export abstract class AstNode implements Locatable {
     public abstract kind: AstNodeKind;
     /**
-     * Where this node starts, counted from the start of its source (see `Locatable.pos`).
+     * Where this node starts, counted from the start of its source (see `Locatable.startPosition`).
      * Use `util.getLocation()` to get the line/character `Location`
      */
-    public abstract pos: number;
+    public abstract startPosition: number;
     /**
-     * Where this node ends: the gap just after its last character (see `Locatable.end`)
+     * Where this node ends: the gap just after its last character (see `Locatable.endPosition`)
      */
-    public abstract end: number;
+    public abstract endPosition: number;
     /**
      * Info about the source this node was parsed from. `undefined` for synthetic nodes
      */
-    public abstract source: SourceInfo | undefined;
+    public abstract sourceInfo: SourceInfo | undefined;
 
     /**
      * @deprecated use `util.getLocation(node)`. This is computed on every read
@@ -170,13 +170,13 @@ export abstract class AstNode implements Locatable {
         //a position only means something within one source (the one this search starts from), so only nodes from that source can match.
         //Nodes without a source (i.e. a synthetic `if` wrapping real code) or from another source (i.e. code parsed from a snippet) never match
         //on their own, but the walk still steps into them, so the real code inside them can still be found
-        const searchSource = this.source;
+        const searchSource = this.sourceInfo;
         return this.findChild<TNodeType>((node) => {
-            if (!node?.source || (searchSource && node.source !== searchSource)) {
+            if (!node?.sourceInfo || (searchSource && node.sourceInfo !== searchSource)) {
                 return;
             }
-            const offset = util.getOffset(node.source, position);
-            if (offset !== undefined && node.pos <= offset && offset <= node.end) {
+            const offset = util.getOffset(node.sourceInfo, position);
+            if (offset !== undefined && node.startPosition <= offset && offset <= node.endPosition) {
                 return node.findChildAtPosition(position, options) ?? node;
             }
         }, options);