fix!: enforce Python logical line boundaries

Cargo.toml

@@ -25,4 +25,4 @@ tree-sitter-language = "0.1"
 cc = "1.2"
 
 [dev-dependencies]
-tree-sitter = { package = "willbooster-tree-sitter", version = "13.0.0" }
+tree-sitter = { package = "willbooster-tree-sitter", version = "14.0.0" }

README.md

@@ -15,14 +15,14 @@ to its authors and contributors. This is not an official release of that project
 This fork builds on upstream v0.25.0 with grammar, scanner, and highlighting fixes, including Python 3.15
 unpacking comprehensions. It ships a compact parser generated by the parser generator of
 [WillBooster/tree-sitter](https://github.com/WillBooster/tree-sitter). The parser carries that fork's own ABI number, so
-it loads only with the WillBooster runtime (`willbooster-tree-sitter` or `@willbooster/web-tree-sitter`) 13.0.0 or later within 13.x,
+it loads only with the WillBooster runtime (`willbooster-tree-sitter` or `@willbooster/web-tree-sitter`) 14.0.0 or later within 14.x,
 not with upstream's tree-sitter.
 
 ## Usage
 
 The npm package ships `tree-sitter-python.wasm` for
 [@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter), which runs in Node.js, Bun,
-browsers, and Cloudflare Workers. The parser requires runtime 13.0.0 or later within 13.x.
+browsers, and Cloudflare Workers. The parser requires runtime 14.0.0 or later within 14.x.
 
 In Node.js and Bun, load the grammar from its path:
 
@@ -67,12 +67,12 @@ Qualified generic bases use `member_type` inside `generic_type`. Generic bases i
 
 In Rust, depend on the [crate](https://crates.io/crates/willbooster-tree-sitter-python) and on
 [willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter), the runtime this package is tested and
-fuzzed with. The parser requires runtime 13.0.0 or later within 13.x:
+fuzzed with. The parser requires runtime 14.0.0 or later within 14.x:
 
 ```toml
 [dependencies]
-tree-sitter = { package = "willbooster-tree-sitter", version = "13.0.0" }
-tree-sitter-python = { package = "willbooster-tree-sitter-python", version = "10" }
+tree-sitter = { package = "willbooster-tree-sitter", version = "14.0.0" }
+tree-sitter-python = { package = "willbooster-tree-sitter-python", version = "11" }
 ```
 
 ```rust

bindings/rust/lib.rs

@@ -1,6 +1,6 @@
 //! This crate provides Python language support for the [tree-sitter] parsing library.
 //!
-//! The parser loads only with the WillBooster runtime, the [willbooster-tree-sitter] crate 13.0.0 or later within 13.x.
+//! The parser loads only with the WillBooster runtime, the [willbooster-tree-sitter] crate 14.0.0 or later within 14.x.
 //!
 //! Typically, you will use the [`LANGUAGE`] constant to add this language to a
 //! tree-sitter [`Parser`], and then use the parser to parse some code:
@@ -16,7 +16,7 @@
 //! assert!(!tree.root_node().has_error());
 //! ```
 //!
-//! [`Parser`]: https://docs.rs/willbooster-tree-sitter/13/tree_sitter/struct.Parser.html
+//! [`Parser`]: https://docs.rs/willbooster-tree-sitter/14.0.0/tree_sitter/struct.Parser.html
 //! [tree-sitter]: https://tree-sitter.github.io/
 //! [willbooster-tree-sitter]: https://crates.io/crates/willbooster-tree-sitter
 

grammar.js

@@ -40,7 +40,7 @@ const SEMICOLON = ';';
 module.exports = grammar({
   name: 'python',
 
-  extras: ($) => [$.comment, /[\s\f\uFEFF\u2060\u200B]|\r?\n/, $.line_continuation],
+  extras: ($) => [$.comment, /[ \t\f\uFEFF\u2060\u200B]/, sym('line_continuation'), sym('_line_break')],
 
   conflicts: ($) => [
     [$.default_type_parameter, $._default_type_primary, $.primary_expression],
@@ -76,11 +76,6 @@ module.exports = grammar({
     sym('escape_interpolation'),
     sym('string_end'),
 
-    // Mark comments as external tokens so that the external scanner is always
-    // invoked, even if no external token is expected. This allows for better
-    // error recovery, because the external scanner can maintain the overall
-    // structure by returning dedent tokens whenever a dedent occurs, even
-    // if no dedent is expected.
     sym('comment'),
 
     '(',
@@ -91,6 +86,8 @@ module.exports = grammar({
     '}',
     'except',
     sym('_format_specifier_content'),
+    sym('_line_break'),
+    sym('line_continuation'),
   ],
 
   inline: ($) => [
@@ -962,7 +959,7 @@ module.exports = grammar({
               /U[a-fA-F\d]{8}/,
               /x[a-fA-F\d]{2}/,
               /\d{1,3}/,
-              /\r?\n/,
+              /\r\n?|\n/,
               /['"abfrntv\\]/,
               /N\{[^}]+\}/
             )
@@ -1022,9 +1019,7 @@ module.exports = grammar({
 
     await: ($) => prec(PREC.unary, seq('await', $.primary_expression)),
 
-    comment: () => token(seq('#', /.*/)),
-
-    line_continuation: () => token(seq('\\', choice(seq(optional('\r'), '\n'), '\0'))),
+    comment: () => token(seq('#', /[^\r\n]*/)),
 
     positional_separator: () => '/',
     keyword_separator: () => '*',

package.json

@@ -47,7 +47,7 @@
     "@willbooster/oxfmt-config": "1.2.2",
     "@willbooster/oxlint-config": "1.4.8",
     "@willbooster/wb": "22.26.2",
-    "@willbooster/web-tree-sitter": "13.0.0",
+    "@willbooster/web-tree-sitter": "14.0.0",
     "conventional-changelog-conventionalcommits": "10.4.1",
     "lefthook": "2.2.2",
     "oxfmt": "0.73.0",
@@ -61,7 +61,7 @@
     "wrangler": "4.149.0"
   },
   "peerDependencies": {
-    "@willbooster/web-tree-sitter": "^13.0.0"
+    "@willbooster/web-tree-sitter": "^14.0.0"
   },
   "peerDependenciesMeta": {
     "@willbooster/web-tree-sitter": {

src/grammar.json

@@ -6588,7 +6588,7 @@
                 },
                 {
                   "type": "PATTERN",
-                  "value": "\\r?\\n"
+                  "value": "\\r\\n?|\\n"
                 },
                 {
                   "type": "PATTERN",
@@ -7076,49 +7076,7 @@
           },
           {
             "type": "PATTERN",
-            "value": ".*"
-          }
-        ]
-      }
-    },
-    "line_continuation": {
-      "type": "TOKEN",
-      "content": {
-        "type": "SEQ",
-        "members": [
-          {
-            "type": "STRING",
-            "value": "\\"
-          },
-          {
-            "type": "CHOICE",
-            "members": [
-              {
-                "type": "SEQ",
-                "members": [
-                  {
-                    "type": "CHOICE",
-                    "members": [
-                      {
-                        "type": "STRING",
-                        "value": "\r"
-                      },
-                      {
-                        "type": "BLANK"
-                      }
-                    ]
-                  },
-                  {
-                    "type": "STRING",
-                    "value": "\n"
-                  }
-                ]
-              },
-              {
-                "type": "STRING",
-                "value": "\u0000"
-              }
-            ]
+            "value": "[^\\r\\n]*"
           }
         ]
       }
@@ -7139,11 +7097,15 @@
     },
     {
       "type": "PATTERN",
-      "value": "[\\s\\f\\uFEFF\\u2060\\u200B]|\\r?\\n"
+      "value": "[ \\t\\f\\uFEFF\\u2060\\u200B]"
     },
     {
       "type": "SYMBOL",
       "name": "line_continuation"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_line_break"
     }
   ],
   "conflicts": [
@@ -7266,6 +7228,14 @@
     {
       "type": "SYMBOL",
       "name": "_format_specifier_content"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_line_break"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "line_continuation"
     }
   ],
   "inline": [