Add arrow function expressions (#1929)

docs/arrow-functions.md

@@ -0,0 +1,124 @@
+# Arrow Functions
+
+Arrow functions are a short way to write a function whose body is a single expression, modeled on the `=>` syntax from TypeScript/JavaScript. They are most useful for small callbacks passed to other functions.
+
+```brighterscript
+adults = utils.filter(people, (p) => p.age >= 18)
+```
+
+transpiles to
+
+```brightscript
+adults = utils_filter(people, function(p)
+    return p.age >= 18
+end function)
+```
+
+## Syntax
+
+An arrow function is a parenthesized parameter list, an optional return type, `=>`, and a single expression that becomes the function's return value. Parameters use the same syntax as regular functions, so types, optional parameters and default values all work.
+
+```brighterscript
+' no params
+getZero = () => 0
+
+' single param (parens are required, even for one param)
+double = (x) => x * 2
+
+' multiple params
+add = (a, b) => a + b
+
+' typed params and return type
+add = (a as integer, b as integer) as integer => a + b
+
+' optional params with default values
+greet = (name = "world") => "hello " + name
+
+' passed as an argument
+names = map(people, (p) => p.name)
+
+' returning an associative array literal (wrap it in parens)
+makePoint = (x, y) => ({ x: x, y: y })
+```
+
+The body extends as far to the right as possible, so `(a) => a ? 1 : 2` returns the whole ternary expression.
+
+## Transpilation
+
+An arrow function compiles to an ordinary anonymous function that returns its expression:
+
+```brighterscript
+add = (a as integer, b as integer) as integer => a + b
+```
+
+```brightscript
+add = function(a as integer, b as integer) as integer
+    return a + b
+end function
+```
+
+When the body is a call to something that returns nothing (a `sub`, or a built-in method like `push`), the arrow function compiles to a `sub` whose body is a plain statement, since `return list.push(item)` is not valid on device:
+
+```brighterscript
+addItem = (list as integer[], item as integer) => list.push(item)
+```
+
+```brightscript
+addItem = sub(list as dynamic, item as integer)
+    list.push(item)
+end sub
+```
+
+## Type checking
+
+Arrow functions are type-checked exactly like anonymous functions. They can be used anywhere a [function type](function-types.md) is expected, and their arguments and return values are validated.
+
+Without an explicit return type, the return type is inferred from the body expression:
+
+```brighterscript
+sub apply(callback as function(x as integer) as integer)
+end sub
+
+' ok
+apply((x as integer) => x * 2)
+
+' error: "function (x as string) as string" is not compatible with "function (x as integer) as integer"
+apply((x as string) => x + "!")
+```
+
+## No closures (yet)
+
+Arrow functions do **not** capture variables from the function they are written in. They follow the same scoping rules as regular anonymous functions: they can see parameters, file-level functions, namespaces, classes and globals, but not the locals of the enclosing function. Referencing one is a compile error:
+
+```brighterscript
+sub main()
+    factor = 2
+    ' error: Arrow functions cannot use variable 'factor' from the enclosing function
+    scale = (x) => x * factor
+
+    ' pass it in as a parameter instead
+    scale = (x, by) => x * by
+    print scale(10, factor)
+end sub
+```
+
+`m` is also not allowed inside an arrow function. Pass the value you need in as an argument instead:
+
+```brighterscript
+' error: 'm' cannot be used inside an arrow function
+render = (data) => m.view.setData(data)
+
+' ok
+render = (data, view) => view.setData(data)
+render(data, m.view)
+```
+
+Both restrictions are deliberate. Closures are being designed separately (see [#530](https://github.com/rokucommunity/brighterscript/issues/530)) and will most likely reuse the `=>` syntax, so what "capturing a variable" and `m` mean inside an arrow function is left open until then. Code that works today will keep working unchanged once closures exist.
+
+An arrow function that captures nothing and does not use `m` is a plain BrightScript function. It can be passed anywhere a function can: parameters typed `as function` or with a function type, untyped (`dynamic`) parameters, and compiled `.brs` libraries that simply call `callback(x)`.
+
+## Limitations
+
+- The body must be a single expression. For multiple statements, use a regular `function ... end function`.
+- Parameters must be in parentheses, even when there is only one: `(x) => x * 2`, not `x => x * 2`.
+- Arrow functions are a BrighterScript feature and are not available in `.brs` files.

docs/readme.md

@@ -14,6 +14,13 @@ sub doSomething()
 end sub
 ```
 
+## [Arrow Functions](arrow-functions.md)
+
+```brighterscript
+' a short way to write a single-expression callback
+adults = filter(people, (p) => p.age >= 18)
+```
+
 ## [Assignments](assigments.md)
 
 ```brighterscript

src/CrossScopeValidator.ts

@@ -988,6 +988,11 @@ export class CrossScopeValidator {
             }
         }
 
+        //an arrow function reaching for a local of its enclosing function gets a diagnostic that says why the name is unavailable
+        const arrowFunctionDiagnostic = util.getArrowFunctionCaptureDiagnostic(typeChainResult.astNode);
+        if (arrowFunctionDiagnostic) {
+            return arrowFunctionDiagnostic;
+        }
         if (isCallExpression(typeChainResult.astNode?.parent) && typeChainResult.astNode?.parent.callee === typeChainResult.astNode) {
             return DiagnosticMessages.cannotFindFunction(typeChainResult.itemName, typeChainResult.fullNameOfItem, typeChainResult.itemParentTypeName, parentDescriptor);
         }

src/DiagnosticMessages.ts

@@ -1401,6 +1401,30 @@ export let DiagnosticMessages = {
         message: `Function has ${count} labels used by goto, max is ${max}.`,
         severity: DiagnosticSeverity.Error,
         code: 'exceeds-max-goto-label-count'
+    }),
+    /**
+     * An arrow function references a local variable of the function it is written inside. Arrow functions are plain
+     * functions (closures are not supported yet, see #530), so that variable is not visible to them
+     * @param name the name of the variable being referenced
+     */
+    arrowFunctionCannotCaptureVariable: (name: string) => ({
+        message: `Arrow functions cannot use variable '${name}' from the enclosing function. Capturing outer variables is not supported yet, so pass it in as a parameter instead`,
+        severity: DiagnosticSeverity.Error,
+        code: 'arrow-function-cannot-capture-variable'
+    }),
+    /**
+     * `m` is not allowed inside arrow functions: what it should mean there (BrightScript's call-site `m` or a captured outer `m`)
+     * is left open until closures are designed, so allowing either now would lock that decision in
+     */
+    arrowFunctionCannotUseM: () => ({
+        message: `'m' cannot be used inside an arrow function. Pass the value in as a parameter instead`,
+        severity: DiagnosticSeverity.Error,
+        code: 'arrow-function-cannot-use-m'
+    }),
+    expectedExpressionAfterArrow: () => ({
+        message: `Expected an expression after '=>'`,
+        severity: DiagnosticSeverity.Error,
+        code: 'expected-expression-after-arrow'
     })
 };
 export const defaultMaximumTruncationLength = 160;

src/bscPlugin/validation/BrsFileValidator.ts

@@ -366,7 +366,8 @@ export class BrsFileValidator {
                     // symbol table should not include any symbols from parent func
                     funcSymbolTable.pushParentProvider(() => node.findAncestor<Body>(isBody).getSymbolTable());
                 }
-                if (!funcSymbolTable?.hasSymbol('m', SymbolTypeFlag.runtime) || isInlineFunc) {
+                //arrow functions don't get an `m`: what it should mean there is left open until closures are designed (see #1929)
+                if (!node.isArrowFunction && (!funcSymbolTable?.hasSymbol('m', SymbolTypeFlag.runtime) || isInlineFunc)) {
                     if (!isTypecastStatement(node.body?.statements?.[0])) {
                         // if this is an inline function, or if the function body does not start with a typecast statement, add the `m` symbol to the function scope. If the function body starts with a typecast statement, the `m` symbol will be added in the visitor for TypecastStatement, and it will be typed as the type from the typecast statement
                         funcSymbolTable?.addSymbol('m', { isInstance: true }, new AssociativeArrayType(), SymbolTypeFlag.runtime);
@@ -377,6 +378,9 @@ export class BrsFileValidator {
                 this.collectFunctionLiterals(node);
                 this.validateFunctionGotoLabelCount(node);
                 this.flagControlFlowUnreachableCode(node);
+                if (node.isArrowFunction) {
+                    this.validateArrowFunctionDoesNotUseM(node);
+                }
             },
             FunctionParameterExpression: (node) => {
                 if (isTypedFunctionTypeExpression(node.parent)) {
@@ -459,6 +463,11 @@ export class BrsFileValidator {
             },
             ReturnStatement: (node) => {
                 const func = node.findAncestor(isFunctionExpression);
+                if (func?.isArrowFunction) {
+                    //an arrow function's `return` is synthesized from its body expression so it always has a value;
+                    //whether that value fits the return type is checked by ScopeValidator.validateReturnStatement
+                    return;
+                }
                 //these situations cannot have a value next to `return`
                 if (
                     //`function as void`, `sub as void`
@@ -753,6 +762,29 @@ export class BrsFileValidator {
         }
     }
 
+    /**
+     * `m` is not allowed inside arrow functions: what it should mean there (BrightScript's call-site `m` or a captured outer `m`)
+     * is left open until closures are designed (see #1929). Checked per arrow function rather than for every `m` in the file, so
+     * only arrow function bodies pay for it. Does not step into nested functions: a nested arrow function is checked on its own
+     * visit, and a nested regular function may use `m` as usual
+     */
+    private validateArrowFunctionDoesNotUseM(arrow: FunctionExpression) {
+        const visitor = createVisitor({
+            VariableExpression: (node) => {
+                if (node.tokens.name?.text?.toLowerCase() === 'm') {
+                    this.event.program.diagnostics.register({
+                        ...DiagnosticMessages.arrowFunctionCannotUseM(),
+                        location: node.tokens.name.location
+                    });
+                }
+            }
+        });
+        for (const param of arrow.parameters) {
+            param.walk(visitor, { walkMode: WalkMode.visitExpressions });
+        }
+        arrow.body?.walk(visitor, { walkMode: WalkMode.visitExpressions });
+    }
+
     private validateFunctionParameterCount(func: FunctionExpression) {
         if (func.parameters.length > CallExpression.MaximumArguments) {
             //flag every parameter over the limit

src/bscPlugin/validation/ScopeValidator.ts

@@ -891,7 +891,13 @@ export class ScopeValidator {
     private validateReturnStatement(file: BrsFile, returnStmt: ReturnStatement) {
         const data: ExtraSymbolData = {};
         const getTypeOptions = { flags: SymbolTypeFlag.runtime, data: data };
-        let funcType = returnStmt.findAncestor(isFunctionExpression)?.getType({ flags: SymbolTypeFlag.typetime });
+        const func = returnStmt.findAncestor<FunctionExpression>(isFunctionExpression);
+        let funcType = func?.getType({ flags: SymbolTypeFlag.typetime });
+        if (func?.isArrowFunction && isTypedFunctionType(funcType) && isVoidType(funcType.returnType) && isCallExpression(returnStmt.value)) {
+            //a void arrow function (`(x) => list.push(x)`) transpiles to a `sub` that runs its body as a plain statement,
+            //so there is no return value to check
+            return;
+        }
         if (isTypedFunctionType(funcType)) {
             let actualReturnType = returnStmt?.value
                 ? this.getNodeTypeWrapper(file, returnStmt?.value, getTypeOptions)
@@ -1271,7 +1277,13 @@ export class ScopeValidator {
                 // only show "cannot find... " errors if the type is not defined from a doc comment
                 const typeChainScan = util.processTypeChain(typeChain);
                 const circularReferenceInfo = this.getCircularReference(exprType);
-                if (isCallExpression(typeChainScan.astNode.parent) && typeChainScan.astNode.parent.callee === expression) {
+                const arrowFunctionDiagnostic = util.getArrowFunctionCaptureDiagnostic(typeChainScan.astNode);
+                if (arrowFunctionDiagnostic) {
+                    this.addMultiScopeDiagnostic({
+                        ...arrowFunctionDiagnostic,
+                        location: typeChainScan?.location
+                    });
+                } else if (isCallExpression(typeChainScan.astNode.parent) && typeChainScan.astNode.parent.callee === expression) {
                     this.addMultiScopeDiagnostic({
                         ...DiagnosticMessages.cannotFindFunction(typeChainScan.itemName, typeChainScan.fullNameOfItem, typeChainScan.itemParentTypeName, this.getParentTypeDescriptor(typeChainScan)),
                         location: typeChainScan?.location

src/lexer/Lexer.spec.ts

@@ -42,6 +42,36 @@ describe('lexer', () => {
         ]);
     });
 
+    it('recognizes the arrow operator without disturbing equals and greater-than', () => {
+        expectKinds('=> = > >= = >', [
+            TokenKind.Arrow,
+            TokenKind.Equal,
+            TokenKind.Greater,
+            TokenKind.GreaterEqual,
+            TokenKind.Equal,
+            TokenKind.Greater
+        ]);
+    });
+
+    it('recognizes the arrow operator without surrounding whitespace', () => {
+        expectKinds('(x)=>x*2', [
+            TokenKind.LeftParen,
+            TokenKind.Identifier,
+            TokenKind.RightParen,
+            TokenKind.Arrow,
+            TokenKind.Identifier,
+            TokenKind.Star,
+            TokenKind.IntegerLiteral
+        ]);
+    });
+
+    it('gives the arrow token its text and location', () => {
+        const tokens = Lexer.scan('a => b').tokens;
+        const arrow = tokens.find(x => x.kind === TokenKind.Arrow);
+        expect(arrow.text).to.eql('=>');
+        expect(arrow.location.range).to.eql(Range.create(0, 2, 0, 4));
+    });
+
     it('separates optional chain characters and LeftSquare when found at beginning of statement locations', () => {
         //a statement starting with a question mark is actually a print statement, so we need to keep the ? separate from [
         expectKinds(`?[ ?[ : ?[ ?[`, [

src/lexer/Lexer.ts

@@ -275,6 +275,14 @@ export class Lexer {
                     break;
             }
         },
+        '=': function (this: Lexer) {
+            if (this.peek() === '>') {
+                this.advance();
+                this.addToken(TokenKind.Arrow);
+            } else {
+                this.addToken(TokenKind.Equal);
+            }
+        },
         '>': function (this: Lexer) {
             switch (this.peek()) {
                 case '=':
@@ -348,7 +356,6 @@ export class Lexer {
     private static tokenKindMap = {
         '(': TokenKind.LeftParen,
         ')': TokenKind.RightParen,
-        '=': TokenKind.Equal,
         ',': TokenKind.Comma,
         '{': TokenKind.LeftCurlyBrace,
         '}': TokenKind.RightCurlyBrace,

src/lexer/TokenKind.ts

@@ -42,6 +42,7 @@ export enum TokenKind {
     GreaterEqual = 'GreaterEqual', // >=
     Equal = 'Equal', // =
     LessGreater = 'LessGreater', // BrightScript uses `<>` for "not equal"
+    Arrow = 'Arrow', // => (arrow function)
 
     // literals
     Identifier = 'Identifier',
@@ -836,6 +837,7 @@ export const FixedTokenText: Partial<Record<TokenKind, string>> = {
     [TokenKind.GreaterEqual]: '>=',
     [TokenKind.Equal]: '=',
     [TokenKind.LessGreater]: '<>',
+    [TokenKind.Arrow]: '=>',
     [TokenKind.Dot]: '.',
     [TokenKind.DotDotDot]: '...',
     [TokenKind.Comma]: ',',