Add lexer and parser sources

2025-06-14 23:57:48 +02:00
parent d5e2d53e9b
commit f524311f06
25 changed files with 3475 additions and 427 deletions
--- a/source/Lexer.def
+++ b/source/Lexer.def
@ -0,0 +1,107 @@
+DEFINITION MODULE Lexer;
+
+FROM FIO IMPORT File;
+
+FROM Common IMPORT Identifier, ShortString, TextLocation;
+
+TYPE
+  PLexerBuffer = POINTER TO CHAR;
+  BufferPosition = RECORD
+    iterator: PLexerBuffer;
+    location: TextLocation
+  END;
+  PBufferPosition = POINTER TO BufferPosition;
+  Lexer = RECORD
+    input: File;
+    buffer: PLexerBuffer;
+    size: CARDINAL;
+    length: CARDINAL;
+    start: BufferPosition;
+    current: BufferPosition
+  END;
+  PLexer = POINTER TO Lexer;
+  LexerKind = (
+    lexerKindEof,
+    lexerKindIdentifier,
+    lexerKindIf,
+    lexerKindThen,
+    lexerKindElse,
+    lexerKindElsif,
+    lexerKindWhile,
+    lexerKindDo,
+    lexerKindProc,
+    lexerKindBegin,
+    lexerKindEnd,
+    lexerKindXor,
+    lexerKindConst,
+    lexerKindVar,
+    lexerKindCase,
+    lexerKindOf,
+    lexerKindType,
+    lexerKindRecord,
+    lexerKindUnion,
+    lexerKindPipe,
+    lexerKindTo,
+    lexerKindBoolean,
+    lexerKindNull,
+    lexerKindAnd,
+    lexerKindOr,
+    lexerKindTilde,
+    lexerKindReturn,
+    lexerKindDefer,
+    lexerKindRange,
+    lexerKindLeftParen,
+    lexerKindRightParen,
+    lexerKindLeftSquare,
+    lexerKindRightSquare,
+    lexerKindGreaterEqual,
+    lexerKindLessEqual,
+    lexerKindGreaterThan,
+    lexerKindLessThan,
+    lexerKindNotEqual,
+    lexerKindEqual,
+    lexerKindSemicolon,
+    lexerKindDot,
+    lexerKindComma,
+    lexerKindPlus,
+    lexerKindMinus,
+    lexerKindAsterisk,
+    lexerKindDivision,
+    lexerKindRemainder,
+    lexerKindAssignment,
+    lexerKindColon,
+    lexerKindHat,
+    lexerKindAt,
+    lexerKindComment,
+    lexerKindInteger,
+    lexerKindWord,
+    lexerKindCharacter,
+    lexerKindString,
+    lexerKindFrom,
+    lexerKindPointer,
+    lexerKindArray,
+    lexerKindArrow,
+    lexerKindProgram,
+    lexerKindModule,
+    lexerKindImport
+  );
+  LexerToken = RECORD
+    CASE kind: LexerKind OF
+      lexerKindBoolean: booleanKind: BOOLEAN |
+      lexerKindIdentifier: identifierKind: Identifier |
+      lexerKindInteger: integerKind: INTEGER |
+      lexerKindString: stringKind: ShortString
+    END;
+    start_location: TextLocation;
+    end_location: TextLocation
+  END;
+  PLexerToken = POINTER TO LexerToken;
+
+PROCEDURE lexer_initialize(lexer: PLexer; input: File);
+PROCEDURE lexer_destroy(lexer: PLexer);
+(* Returns the last read token. *)
+PROCEDURE lexer_current(lexer: PLexer): LexerToken;
+(* Read and return the next token. *)
+PROCEDURE lexer_lex(lexer: PLexer): LexerToken;
+
+END Lexer.