belka/elna
1
0
Fork 0
Code Issues 5 Pull Requests Releases Activity

Trace the source code position in the lexer

Browse Source
This commit is contained in:
Eugen Wissner
2025-06-12 18:44:06 +02:00
parent 90aa5a0030
commit fdaeb25f73
7 changed files with 252 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
source/Parser.mod
View File

@ -9,7 +9,7 @@ FROM Storage IMPORT ALLOCATE, REALLOCATE;
FROM Lexer IMPORT Lexer, LexerKind, LexerToken, lexer_current, lexer_lex;
(* Calls lexer_lex() but skips the comments. *)
PROCEDURE transpiler_lex(lexer: PLexer): LexerToken;
PROCEDURE parser_lex(lexer: PLexer): LexerToken;
VAR
result: LexerToken;
BEGIN
@ -20,7 +20,7 @@ BEGIN
END;
RETURN result
END transpiler_lex;
END parser_lex;
PROCEDURE parse_type_fields(lexer: PLexer): PAstFieldDeclaration;
VAR
token: LexerToken;
@ -29,7 +29,7 @@ VAR
current_field: PAstFieldDeclaration;
BEGIN
ALLOCATE(field_declarations, TSIZE(AstFieldDeclaration));
token := transpiler_lex(lexer);
token := parser_lex(lexer);
field_count := 0;
WHILE token.kind <> lexerKindEnd DO
@ -40,16 +40,16 @@ BEGIN
current_field := field_declarations;
INC(current_field , TSIZE(AstFieldDeclaration) * (field_count - 1));
token := transpiler_lex(lexer);
token := parser_lex(lexer);
current_field^.field_name := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
current_field^.field_type := parse_type_expression(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
IF token.kind = lexerKindSemicolon THEN
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END
END;
INC(current_field, TSIZE(AstFieldDeclaration));
@ -78,7 +78,7 @@ BEGIN
token := lexer_current(lexer);
IF token.kind = lexerKindPointer THEN
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
token := lexer_current(lexer);
result^.target := parse_type_expression(lexer);
@ -98,16 +98,16 @@ BEGIN
token := lexer_current(lexer);
IF token.kind = lexerKindArray THEN
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
IF token.kind <> lexerKindOf THEN
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.length := token.integerKind;
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.base := parse_type_expression(lexer);
RETURN result
@ -124,14 +124,14 @@ BEGIN
case_count := 1;
ALLOCATE(result^.cases, TSIZE(Identifier) * 2);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
current_case := result^.cases;
current_case^ := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind = lexerKindComma DO
token := transpiler_lex(lexer);
token := parser_lex(lexer);
INC(case_count);
INC(case_count);
@ -141,7 +141,7 @@ BEGIN
INC(current_case, TSIZE(Identifier) * (case_count - 1));
current_case^ := token.identifierKind;
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
INC(current_case, TSIZE(Identifier));
MemZero(current_case, TSIZE(Identifier));
@ -174,8 +174,8 @@ BEGIN
ALLOCATE(result^.parameters, 1);
token := transpiler_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind <> lexerKindRightParen DO
INC(parameter_count);
@ -187,9 +187,9 @@ BEGIN
current_parameter^ := parse_type_expression(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
IF token.kind = lexerKindComma THEN
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END
END;
current_parameter := result^.parameters;
@ -236,11 +236,11 @@ BEGIN
NEW(result);
result^.identifier := token.identifierKind;
token := transpiler_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := parser_lex(lexer);
result^.type_expression := parse_type_expression(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_type_declaration;
@ -258,7 +258,7 @@ BEGIN
declaration_count := 0;
IF token.kind = lexerKindType THEN
token := transpiler_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind = lexerKindIdentifier DO
INC(declaration_count);
@ -268,7 +268,7 @@ BEGIN
INC(current_declaration, TSIZE(PAstTypedDeclaration) * (declaration_count - 1));
current_declaration^ := parse_type_declaration(lexer);
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END
END;
IF declaration_count <> 0 THEN
@ -288,12 +288,12 @@ BEGIN
token := lexer_current(lexer);
result^.variable_name := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.variable_type := parse_type_expression(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_variable_declaration;
PROCEDURE parse_variable_part(lexer: PLexer): PPAstVariableDeclaration;
@ -310,7 +310,7 @@ BEGIN
declaration_count := 0;
IF token.kind = lexerKindVar THEN
token := transpiler_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind = lexerKindIdentifier DO
INC(declaration_count);
@ -320,7 +320,7 @@ BEGIN
INC(current_declaration, TSIZE(PAstVariableDeclaration) * (declaration_count - 1));
current_declaration^ := parse_variable_declaration(lexer);
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END
END;
IF declaration_count <> 0 THEN
@ -340,12 +340,12 @@ BEGIN
token := lexer_current(lexer);
result^.constant_name := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.constant_value := token.integerKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_constant_declaration;
@ -363,7 +363,7 @@ BEGIN
declaration_count := 0;
IF token.kind = lexerKindConst THEN
token := transpiler_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind = lexerKindIdentifier DO
INC(declaration_count);
@ -373,7 +373,7 @@ BEGIN
INC(current_declaration, TSIZE(PAstConstantDeclaration) * (declaration_count - 1));
current_declaration^ := parse_constant_declaration(lexer);
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END
END;
IF declaration_count <> 0 THEN
@ -393,20 +393,20 @@ BEGIN
NEW(result);
symbol_count := 1;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.package := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
ALLOCATE(result^.symbols, TSIZE(Identifier) * 2);
current_symbol := result^.symbols;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
current_symbol^ := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind <> lexerKindSemicolon DO
token := transpiler_lex(lexer);
token := parser_lex(lexer);
INC(symbol_count);
REALLOCATE(result^.symbols, TSIZE(Identifier) * (symbol_count + 1));
@ -414,12 +414,12 @@ BEGIN
INC(current_symbol, TSIZE(Identifier) * (symbol_count - 1));
current_symbol^ := token.identifierKind;
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
INC(current_symbol, TSIZE(Identifier));
MemZero(current_symbol, TSIZE(Identifier));
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_import_statement;
@ -484,7 +484,7 @@ BEGIN
literal^.boolean := token.booleanKind
END;
IF literal <> NIL THEN
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
RETURN literal
@ -508,7 +508,7 @@ BEGIN
END;
IF (result = NIL) AND (next_token.kind = lexerKindMinus) THEN
NEW(result);
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
result^.kind := astExpressionKindUnary;
result^.unary_operator := astUnaryOperatorMinus;
@ -516,17 +516,17 @@ BEGIN
END;
IF (result = NIL) AND (next_token.kind = lexerKindTilde) THEN
NEW(result);
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
result^.kind := astExpressionKindUnary;
result^.unary_operator := astUnaryOperatorNot;
result^.unary_operand := parse_factor(lexer)
END;
IF (result = NIL) AND (next_token.kind = lexerKindLeftParen) THEN
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
result := parse_expression(lexer);
IF result <> NIL THEN
next_token := transpiler_lex(lexer)
next_token := parser_lex(lexer)
END
END;
IF (result = NIL) AND (next_token.kind = lexerKindIdentifier) THEN
@ -535,7 +535,7 @@ BEGIN
result^.kind := astExpressionKindIdentifier;
result^.identifier := next_token.identifierKind;
next_token := transpiler_lex(lexer)
next_token := parser_lex(lexer)
END;
RETURN result
@ -562,34 +562,34 @@ BEGIN
designator^.kind := astExpressionKindDereference;
designator^.reference := inner_expression;
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
handled := TRUE
END;
IF ~handled AND (next_token.kind = lexerKindLeftSquare) THEN
NEW(designator);
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
designator^.kind := astExpressionKindArrayAccess;
designator^.array := inner_expression;
designator^.index := parse_expression(lexer);
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
handled := TRUE
END;
IF ~handled AND (next_token.kind = lexerKindDot) THEN
NEW(designator);
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
designator^.kind := astExpressionKindFieldAccess;
designator^.aggregate := inner_expression;
designator^.field := next_token.identifierKind;
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
handled := TRUE
END;
IF ~handled AND (next_token.kind = lexerKindLeftParen) THEN
NEW(designator);
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
designator^.kind := astExpressionKindCall;
designator^.callable := inner_expression;
@ -604,7 +604,7 @@ BEGIN
next_token := lexer_current(lexer);
WHILE next_token.kind = lexerKindComma DO
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
designator^.argument_count := designator^.argument_count + 1;
REALLOCATE(designator^.arguments, TSIZE(PAstExpression) * designator^.argument_count);
@ -616,7 +616,7 @@ BEGIN
END
END;
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
handled := TRUE
END
END;
@ -629,7 +629,7 @@ VAR
result: PAstExpression;
right: PAstExpression;
BEGIN
next_token := transpiler_lex(lexer);
next_token := parser_lex(lexer);
right := parse_designator(lexer);
result := NIL;
@ -703,7 +703,7 @@ BEGIN
NEW(result);
result^.kind := astStatementKindReturn;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.returned := parse_expression(lexer);
RETURN result
@ -717,7 +717,7 @@ BEGIN
result^.kind := astStatementKindAssignment;
result^.assignee := assignee;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.assignment := parse_expression(lexer);
RETURN result
@ -766,7 +766,7 @@ VAR
designator: PAstExpression;
BEGIN
statement := NIL;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
IF token.kind = lexerKindIf THEN
statement := parse_if_statement(lexer)
@ -798,11 +798,11 @@ BEGIN
NEW(result);
result^.kind := astStatementKindIf;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.if_condition := parse_expression(lexer);
result^.if_branch := parse_compound_statement(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_if_statement;
PROCEDURE parse_while_statement(lexer: PLexer): PAstStatement;
@ -813,11 +813,11 @@ BEGIN
NEW(result);
result^.kind := astStatementKindWhile;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.while_condition := parse_expression(lexer);
result^.while_body := parse_compound_statement(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_while_statement;
PROCEDURE parse_statement_part(lexer: PLexer): AstCompoundStatement;
@ -844,15 +844,15 @@ VAR
BEGIN
NEW(declaration);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
declaration^.name := token.identifierKind;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
declaration^.parameters := NIL;
declaration^.parameter_count := 0;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
WHILE token.kind <> lexerKindRightParen DO
parameter_index := declaration^.parameter_count;
INC(declaration^.parameter_count);
@ -863,26 +863,26 @@ BEGIN
current_parameter^.identifier := token.identifierKind;
token := transpiler_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := parser_lex(lexer);
current_parameter^.type_expression := parse_type_expression(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
IF token.kind = lexerKindComma THEN
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END
END;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
declaration^.return_type := NIL;
(* Check for the return type and write it. *)
IF token.kind = lexerKindArrow THEN
token := transpiler_lex(lexer);
token := parser_lex(lexer);
declaration^.return_type := parse_type_expression(lexer);
token := transpiler_lex(lexer)
token := parser_lex(lexer)
END;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
RETURN declaration
END parse_procedure_heading;
@ -897,8 +897,8 @@ BEGIN
declaration^.variables := parse_variable_part(lexer);
declaration^.statements := parse_statement_part(lexer);
token := transpiler_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := parser_lex(lexer);
RETURN declaration
END parse_procedure_declaration;
@ -938,16 +938,16 @@ VAR
result: PAstModule;
BEGIN
NEW(result);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.main := TRUE;
IF token.kind = lexerKindModule THEN
result^.main := FALSE
END;
token := transpiler_lex(lexer);
token := parser_lex(lexer);
(* Write the module body. *)
token := transpiler_lex(lexer);
token := parser_lex(lexer);
result^.imports := parse_import_part(lexer);
result^.constants := parse_constant_part(lexer);
@ -957,8 +957,8 @@ BEGIN
result^.procedures := parse_procedure_part(lexer);
result^.statements := parse_statement_part(lexer);
token := transpiler_lex(lexer);
token := transpiler_lex(lexer);
token := parser_lex(lexer);
token := parser_lex(lexer);
RETURN result
END parse_module;