parser: return SyntaxError when parsing fails

Author: gf712

integration/program.cpp

@@ -110,7 +110,7 @@ class RunPythonProgram : public ::testing::Test
 
 		auto lexer = Lexer::create(std::string(program), "_integration_dummy_.py");
 		parser::Parser p{ lexer };
-		p.parse();
+		ASSERT_TRUE(p.parse().is_ok());
 		p.module()->print_node("");
 		m_bytecode = compiler::compile(p.module(),
 			{},

integration/run_python_tests.sh

@@ -46,4 +46,24 @@ else
     echo $file "... PASSED!"
 fi
 
+# A syntax error must exit non-zero and report the line the parser actually gave
+# up on -- not line 1 -- with a caret under the offending token.
+file=$SCRIPT_DIR/tests/expected_failures/syntax_error_reporting.py
+output=$(timeout 10s $PYTHON_EXECUTABLE $file --gc-frequency $GC_FREQUENCY 2>&1)
+if [ $? -eq 0 ]; then
+    echo $file "... FAILED! (expected a non-zero exit code)"
+    exit_code=1
+elif ! echo "$output" | grep -q '", line 4$'; then
+    echo $file "... FAILED! (expected the error on line 4, got: ${output})"
+    exit_code=1
+elif ! echo "$output" | grep -qF '            ^'; then
+    echo $file "... FAILED! (expected a caret under the ':', got: ${output})"
+    exit_code=1
+elif ! echo "$output" | grep -q '^SyntaxError: invalid syntax$'; then
+    echo $file "... FAILED! (expected a SyntaxError, got: ${output})"
+    exit_code=1
+else
+    echo $file "... PASSED!"
+fi
+
 exit $exit_code

integration/tests/expected_failures/syntax_error_reporting.py

@@ -0,0 +1,5 @@
+x = 1
+y = 2
+
+def foo(:
+    pass

src/ast/optimizers/Optimizers_tests.cpp

@@ -466,7 +466,7 @@ void assert_generates_ast(std::string_view program,
 	parser::Parser p{ lexer };
 	const auto spdlog_level = spdlog::get_level();
 	spdlog::set_level(spdlog::level::debug);
-	p.parse();
+	ASSERT(p.parse().is_ok());
 	spdlog::set_level(spdlog_level);
 
 	if (lvl > compiler::OptimizationLevel::None) {

src/executable/bytecode/BytecodeProgram_tests.cpp

@@ -14,7 +14,7 @@ std::shared_ptr<BytecodeProgram> generate_bytecode(std::string_view program)
 {
 	auto lexer = Lexer::create(std::string(program), "_bytecode_program_tests_.py");
 	parser::Parser p{ lexer };
-	p.parse();
+	ASSERT(p.parse().is_ok());
 
 	auto module = p.module();
 	ASSERT(module);

src/executable/bytecode/codegen/BytecodeGenerator_tests.cpp

@@ -15,7 +15,7 @@ std::shared_ptr<BytecodeProgram> generate_bytecode(std::string_view program)
 {
 	auto lexer = Lexer::create(std::string(program), "_bytecode_generator_tests_.py");
 	parser::Parser p{ lexer };
-	p.parse();
+	ASSERT(p.parse().is_ok());
 
 	auto module = p.module();
 	ASSERT(module);

src/executable/bytecode/codegen/VariablesResolver_tests.cpp

@@ -13,7 +13,7 @@ VariablesResolver::VisibilityMap generate_resolver(std::string_view program)
 {
 	auto lexer = Lexer::create(std::string(program), "_bytecode_generator_tests_.py");
 	parser::Parser p{ lexer };
-	p.parse();
+	ASSERT(p.parse().is_ok());
 
 	auto *module = as<ast::Module>(p.module().get());
 	ASSERT(module);

src/executable/llvm/LLVMGenerator_tests.cpp

@@ -14,7 +14,7 @@ std::shared_ptr<Program> generate_llvm_module(std::string_view program)
 {
 	auto lexer = Lexer::create(std::string(program), "_llvm_backend_tests_.py");
 	parser::Parser p{ lexer };
-	p.parse();
+	ASSERT(p.parse().is_ok());
 
 	auto module = as<ast::Module>(p.module());
 	ASSERT(module);

src/parser/Parser.cpp

@@ -1,7 +1,8 @@
 module;
 #include "core.hpp"
-#include "spdlog/spdlog.h"
+#include "runtime/SourceManager.hpp"
 
+#include "spdlog/spdlog.h"
 #include <gmpxx.h>
 
 
@@ -709,6 +710,7 @@ struct SingleTokenPatternV2 : PatternV2<SingleTokenPatternV2<Patterns...>>
 
 	static std::optional<ResultType> matches_impl(Parser &p)
 	{
+		p.observe_token(p.token_position());
 		if (SingleTokenPattern_<ComposedTypes<Patterns...>>::match(p)) {
 			const auto &t = p.lexer().peek_token(p.token_position());
 			return t.has_value() ? std::make_optional(ResultType{ *t }) : std::nullopt;
@@ -7390,33 +7392,41 @@ struct FilePattern : PatternV2<FilePattern>
 			}
 			return p.module();
 		}
-		size_t idx = 0;
-		auto t = *p.lexer().peek_token(idx);
-		auto begin = t.start().pointer_to_program;
-		auto end = t.end().pointer_to_program;
-		const size_t row = t.start().row;
-		while (row == t.start().row) {
-			end = t.end().pointer_to_program;
-			idx++;
-			t = *p.lexer().peek_token(idx);
-		}
-		std::string line{ begin, end };
-		spdlog::error("Syntax error on line {}: '{}'", row + 1, line);
-		// PARSER_ERROR();
 		return {};
 	}
 };
 
 namespace parser {
-void Parser::parse()
+PyResult<std::shared_ptr<ast::Module>> Parser::parse()
 {
 	auto result = PatternMatchV2<FilePattern>::match(*this);
 	if (result) {
 		auto [module] = *result;
 		m_module = std::move(module);
-		m_module->print_node("");
+		return Ok(m_module);
 	}
-	DEBUG_LOG("Parser return code: {}", result.has_value());
+	std::size_t index = m_furthest_token;
+	std::optional<Token> token = m_lexer.peek_token(index);
+	while (!token.has_value() && index > 0) { token = m_lexer.peek_token(--index); }
+
+	const auto &filename = m_lexer.filename();
+	const auto &program = m_lexer.program();
+	auto lineno = token.has_value() ? token->start().row + 1 : 1;
+	auto offset = token.has_value() ? token->start().column + 1 : 1;
+	const auto line_count =
+		std::max(static_cast<std::size_t>(std::count(program.begin(), program.end(), '\n'))
+					 + (program.empty() || program.back() == '\n' ? 0uz : 1uz),
+			1uz);
+	if (lineno > line_count) {
+		lineno = line_count;
+		offset = SourceManager::the().line(filename, lineno).size() + 1;
+	}
+	const auto text = SourceManager::the().line(filename, lineno);
+	return Err(syntax_error("invalid syntax",
+		SyntaxErrorLocation{ .filename = filename,
+			.lineno = lineno,
+			.offset = offset,
+			.text = std::string{ text } }));
 }
 
 PyResult<std::shared_ptr<ast::Module>> Parser::parse_expression()

src/parser/Parser.cppm

@@ -12,6 +12,7 @@ class Parser
 	std::shared_ptr<ast::Module> m_module;
 	Lexer &m_lexer;
 	std::size_t m_token_position{ 0 };
+	std::size_t m_furthest_token{ m_token_position };
 
   public:
 	struct CacheValue
@@ -67,8 +68,15 @@ class Parser
 	const std::size_t &token_position() const { return m_token_position; }
 	std::size_t &token_position() { return m_token_position; }
 
+	std::size_t furthest_token() const { return m_furthest_token; }
+
+	void observe_token(std::size_t position)
+	{
+		m_furthest_token = std::max(m_furthest_token, position);
+	}
+
 	// parses a file
-	void parse();
+	py::PyResult<std::shared_ptr<ast::Module>> parse();
 
 	// parses an expression used by the builtin `eval` function
 	py::PyResult<std::shared_ptr<ast::Module>> parse_expression();