aboutsummaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
authorEugen Wissner <belka@caraus.de>2026-07-12 13:23:54 +0200
committerEugen Wissner <belka@caraus.de>2026-07-12 13:23:54 +0200
commit768537d6e4b82a19e77b7e44234309ecaf4d65d2 (patch)
treeab22613a16c3a914b16d00a5f988f51d1eadac0d /source
parent14d4977e2ab2409bb7344395ca01d19e49f130f1 (diff)
downloadelna-768537d6e4b82a19e77b7e44234309ecaf4d65d2.tar.gz
Remove unions
Diffstat (limited to 'source')
-rw-r--r--source/cstdio.elna12
-rw-r--r--source/cstdlib.elna12
-rw-r--r--source/cstring.elna17
-rw-r--r--source/lexer.elna1050
-rw-r--r--source/main.elna400
5 files changed, 361 insertions, 1130 deletions
diff --git a/source/cstdio.elna b/source/cstdio.elna
index 95283a1..346d931 100644
--- a/source/cstdio.elna
+++ b/source/cstdio.elna
@@ -34,6 +34,9 @@ extern
proc fwrite*(ptr: Pointer; size: Word; nitems: Word; stream: ^FILE): Word
extern
+proc fputc(c: Int; stream: ^FILE): Word
+extern
+
proc perror(s: ^Char)
extern
@@ -43,4 +46,13 @@ extern
proc putchar(c: Int): Int
extern
+proc sprintf(str: Pointer; format: ^Char; number: Word): Int
+extern
+
+proc fprintf(stream: Pointer; format: ^Char; number: Word): Int
+extern
+
+proc fdopen(fildes: Int; mode: ^Char): Pointer
+extern
+
end.
diff --git a/source/cstdlib.elna b/source/cstdlib.elna
index 696e5dd..32669c1 100644
--- a/source/cstdlib.elna
+++ b/source/cstdlib.elna
@@ -2,22 +2,22 @@
v. 2.0. If a copy of the MPL was not distributed with this file, You can
obtain one at https://mozilla.org/MPL/2.0/. *)
-proc malloc(size: Word): Pointer
+proc malloc*(size: Word): Pointer
extern
-proc free(ptr: Pointer)
+proc free*(ptr: Pointer)
extern
-proc calloc(nmemb: Word; size: Word): Pointer
+proc calloc*(nmemb: Word; size: Word): Pointer
extern
-proc realloc(ptr: Pointer; size: Word): Pointer
+proc realloc*(ptr: Pointer; size: Word): Pointer
extern
-proc atoi(str: ^Char): Int
+proc atoi*(str: ^Char): Int
extern
-proc exit(code: Int): !
+proc exit*(code: Int): !
extern
end.
diff --git a/source/cstring.elna b/source/cstring.elna
index ec5cd7b..e21056a 100644
--- a/source/cstring.elna
+++ b/source/cstring.elna
@@ -2,25 +2,28 @@
v. 2.0. If a copy of the MPL was not distributed with this file, You can
obtain one at https://mozilla.org/MPL/2.0/. *)
-proc memset(ptr: Pointer; c: Int; n: Word): ^Char
+proc memset*(ptr: Pointer; c: Int; n: Word): ^Char
extern
-proc memcpy(dst: Pointer; src: Pointer; n: Word)
+proc memcpy*(dst, src: Pointer; n: Word)
extern
-proc strcmp(s1: ^Char; s2: ^Char): Int
+proc memcmp*(s1, s2: Pointer; n: Word): Int
extern
-proc strncmp(s1: ^Char; s2: ^Char; n: Word): Int
+proc strcmp*(s1: ^Char; s2: ^Char): Int
extern
-proc strncpy(dst: ^Char; src: ^Char; dsize: Word): ^Char
+proc strncmp*(s1: ^Char; s2: ^Char; n: Word): Int
extern
-proc strcpy(dst: ^Char; src: ^Char): ^Char
+proc strncpy*(dst: ^Char; src: ^Char; dsize: Word): ^Char
extern
-proc strlen(ptr: ^Char): Word
+proc strcpy*(dst: ^Char; src: ^Char): ^Char
+extern
+
+proc strlen*(ptr: ^Char): Word
extern
end.
diff --git a/source/lexer.elna b/source/lexer.elna
index bb8c8ef..3b015ad 100644
--- a/source/lexer.elna
+++ b/source/lexer.elna
@@ -2,77 +2,53 @@
v. 2.0. If a copy of the MPL was not distributed with this file, You can
obtain one at https://mozilla.org/MPL/2.0/. *)
-import cstdio, cstring, cctype, cstdlib, common
+import cstdio, common
const
CHUNK_SIZE := 85536u
type
- (*
- * Classification table assigns each possible character to a group (class). All
- * characters of the same group are handled equivalently.
- *
- * Classification:
- *)
- TransitionClass = (
- invalid,
- digit,
- alpha,
- space,
- colon,
- equals,
- left_paren,
- right_paren,
- asterisk,
- underscore,
- single,
- hex,
- zero,
- x,
- eof,
- dot,
- minus,
- single_quote,
- double_quote,
- greater,
- less,
- other
- )
- TransitionState = (
- start,
- colon,
- identifier,
- decimal,
- greater,
- minus,
- left_paren,
- less,
- dot,
- comment,
- closing_comment,
- character,
- string,
- leading_zero,
- decimal_suffix,
- finish
- )
- LexerToken = record
- kind: LexerKind;
- value: union
- booleanKind: Bool;
- identifierKind: Identifier;
- integerKind: Int;
- stringKind: String
- end;
- start_location: TextLocation;
- end_location: TextLocation
+ ElnaLexerState = (
+ start,
+ colon,
+ identifier,
+ decimal,
+ leading_zero,
+ greater,
+ minus,
+ left_paren,
+ less,
+ dot,
+ comment,
+ closing_comment,
+ character,
+ character_escape,
+ string,
+ string_escape,
+ number_sign,
+ trait,
+ finish
+ )
+ ElnaLexerToken* = record
+ kind: ElnaLexerKind
end
- TransitionAction = proc(lexer: ^Lexer; token: ^LexerToken)
- Transition = record
+ ElnaLexerBooleanToken* = record(ElnaLexerToken)
+ value: Bool
+ end
+ ElnaLexerCharacterToken* = record(ElnaLexerToken)
+ value: Char
+ end
+ ElnaLexerStringToken* = record(ElnaLexerToken)
+ value: String
+ end
+ ElnaLexerIntegerToken* = record(ElnaLexerToken)
+ value: Int
+ end
+ TransitionAction = proc(lexer: ^Lexer; token: ^ElnaLexerToken)
+ ElnaLexerTransition = record
action: TransitionAction;
- next_state: TransitionState
+ next_state: ElnaLexerState
end
- TransitionClasses = [22]Transition
BufferPosition* = record
iterator: ^Char;
@@ -86,866 +62,96 @@ type
start: BufferPosition;
current: BufferPosition
end
- LexerKind* = (
- unknown,
- identifier,
- _if,
- _then,
- _else,
- _elsif,
- _while,
- _do,
- _proc,
- _begin,
- _end,
- _extern,
- _const,
- _var,
- _case,
- _of,
- _type,
- _record,
- _union,
- pipe,
- to,
- boolean,
- null,
- and,
- _or,
- _xor,
- not,
- _return,
- _cast,
- shift_left,
- shift_right,
- left_paren,
- right_paren,
- left_square,
- right_square,
- greater_equal,
- less_equal,
- greater_than,
- less_than,
- not_equal,
- equal,
- semicolon,
- dot,
- comma,
- plus,
- minus,
- multiplication,
- division,
- remainder,
- assignment,
- colon,
- hat,
- at,
- comment,
- integer,
- word,
- character,
- string,
+ ElnaLexerKind* = (
+ identifier,
+ _const,
+ _var,
+ _proc,
+ _type,
+ _begin,
+ _end,
+ _if,
+ _then,
+ _else,
+ _elsif,
+ _extern,
+ _record,
+ boolean,
+ null,
+ and,
+ _or,
+ _xor,
+ not,
+ _return,
+ _cast,
+ trait,
+ left_paren,
+ right_paren,
+ left_square,
+ right_square,
+ greater_equal,
+ less_equal,
+ greater_than,
+ less_than,
+ not_equal,
+ equals,
+ semicolon,
+ dot,
+ comma,
+ plus,
+ _import,
+ minus,
+ multiplication,
+ division,
+ remainder,
+ assignment,
+ colon,
+ hat,
+ at,
+ comment,
+ string,
+ character,
+ integer,
+ word,
+ _while,
_defer,
exclamation,
- arrow,
- trait,
- _program,
- _module,
- _import
- )
-
-var
- classification: [128]TransitionClass
- transitions: [16]TransitionClasses
-
-proc initialize_classification()
-var
- i: Word
-begin
- classification[1] := TransitionClass.eof; (* NUL *)
- classification[2] := TransitionClass.invalid; (* SOH *)
- classification[3] := TransitionClass.invalid; (* STX *)
- classification[4] := TransitionClass.invalid; (* ETX *)
- classification[5] := TransitionClass.invalid; (* EOT *)
- classification[6] := TransitionClass.invalid; (* EMQ *)
- classification[7] := TransitionClass.invalid; (* ACK *)
- classification[8] := TransitionClass.invalid; (* BEL *)
- classification[9] := TransitionClass.invalid; (* BS *)
- classification[10] := TransitionClass.space; (* HT *)
- classification[11] := TransitionClass.space; (* LF *)
- classification[12] := TransitionClass.invalid; (* VT *)
- classification[13] := TransitionClass.invalid; (* FF *)
- classification[14] := TransitionClass.space; (* CR *)
- classification[15] := TransitionClass.invalid; (* SO *)
- classification[16] := TransitionClass.invalid; (* SI *)
- classification[17] := TransitionClass.invalid; (* DLE *)
- classification[18] := TransitionClass.invalid; (* DC1 *)
- classification[19] := TransitionClass.invalid; (* DC2 *)
- classification[20] := TransitionClass.invalid; (* DC3 *)
- classification[21] := TransitionClass.invalid; (* DC4 *)
- classification[22] := TransitionClass.invalid; (* NAK *)
- classification[23] := TransitionClass.invalid; (* SYN *)
- classification[24] := TransitionClass.invalid; (* ETB *)
- classification[25] := TransitionClass.invalid; (* CAN *)
- classification[26] := TransitionClass.invalid; (* EM *)
- classification[27] := TransitionClass.invalid; (* SUB *)
- classification[28] := TransitionClass.invalid; (* ESC *)
- classification[29] := TransitionClass.invalid; (* FS *)
- classification[30] := TransitionClass.invalid; (* GS *)
- classification[31] := TransitionClass.invalid; (* RS *)
- classification[32] := TransitionClass.invalid; (* US *)
- classification[33] := TransitionClass.space; (* Space *)
- classification[34] := TransitionClass.single; (* ! *)
- classification[35] := TransitionClass.double_quote; (* " *)
- classification[36] := TransitionClass.other; (* # *)
- classification[37] := TransitionClass.other; (* $ *)
- classification[38] := TransitionClass.single; (* % *)
- classification[39] := TransitionClass.single; (* & *)
- classification[40] := TransitionClass.single_quote; (* ' *)
- classification[41] := TransitionClass.left_paren; (* ( *)
- classification[42] := TransitionClass.right_paren; (* ) *)
- classification[43] := TransitionClass.asterisk; (* * *)
- classification[44] := TransitionClass.single; (* + *)
- classification[45] := TransitionClass.single; (* , *)
- classification[46] := TransitionClass.minus; (* - *)
- classification[47] := TransitionClass.dot; (* . *)
- classification[48] := TransitionClass.single; (* / *)
- classification[49] := TransitionClass.zero; (* 0 *)
- classification[50] := TransitionClass.digit; (* 1 *)
- classification[51] := TransitionClass.digit; (* 2 *)
- classification[52] := TransitionClass.digit; (* 3 *)
- classification[53] := TransitionClass.digit; (* 4 *)
- classification[54] := TransitionClass.digit; (* 5 *)
- classification[55] := TransitionClass.digit; (* 6 *)
- classification[56] := TransitionClass.digit; (* 7 *)
- classification[57] := TransitionClass.digit; (* 8 *)
- classification[58] := TransitionClass.digit; (* 9 *)
- classification[59] := TransitionClass.colon; (* : *)
- classification[60] := TransitionClass.single; (* ; *)
- classification[61] := TransitionClass.less; (* < *)
- classification[62] := TransitionClass.equals; (* = *)
- classification[63] := TransitionClass.greater; (* > *)
- classification[64] := TransitionClass.other; (* ? *)
- classification[65] := TransitionClass.single; (* @ *)
- classification[66] := TransitionClass.alpha; (* A *)
- classification[67] := TransitionClass.alpha; (* B *)
- classification[68] := TransitionClass.alpha; (* C *)
- classification[69] := TransitionClass.alpha; (* D *)
- classification[70] := TransitionClass.alpha; (* E *)
- classification[71] := TransitionClass.alpha; (* F *)
- classification[72] := TransitionClass.alpha; (* G *)
- classification[73] := TransitionClass.alpha; (* H *)
- classification[74] := TransitionClass.alpha; (* I *)
- classification[75] := TransitionClass.alpha; (* J *)
- classification[76] := TransitionClass.alpha; (* K *)
- classification[77] := TransitionClass.alpha; (* L *)
- classification[78] := TransitionClass.alpha; (* M *)
- classification[79] := TransitionClass.alpha; (* N *)
- classification[80] := TransitionClass.alpha; (* O *)
- classification[81] := TransitionClass.alpha; (* P *)
- classification[82] := TransitionClass.alpha; (* Q *)
- classification[83] := TransitionClass.alpha; (* R *)
- classification[84] := TransitionClass.alpha; (* S *)
- classification[85] := TransitionClass.alpha; (* T *)
- classification[86] := TransitionClass.alpha; (* U *)
- classification[87] := TransitionClass.alpha; (* V *)
- classification[88] := TransitionClass.alpha; (* W *)
- classification[89] := TransitionClass.alpha; (* X *)
- classification[90] := TransitionClass.alpha; (* Y *)
- classification[91] := TransitionClass.alpha; (* Z *)
- classification[92] := TransitionClass.single; (* [ *)
- classification[93] := TransitionClass.other; (* \ *)
- classification[94] := TransitionClass.single; (* ] *)
- classification[95] := TransitionClass.single; (* ^ *)
- classification[96] := TransitionClass.underscore; (* _ *)
- classification[97] := TransitionClass.other; (* ` *)
- classification[98] := TransitionClass.hex; (* a *)
- classification[99] := TransitionClass.hex; (* b *)
- classification[100] := TransitionClass.hex; (* c *)
- classification[101] := TransitionClass.hex; (* d *)
- classification[102] := TransitionClass.hex; (* e *)
- classification[103] := TransitionClass.hex; (* f *)
- classification[104] := TransitionClass.alpha; (* g *)
- classification[105] := TransitionClass.alpha; (* h *)
- classification[106] := TransitionClass.alpha; (* i *)
- classification[107] := TransitionClass.alpha; (* j *)
- classification[108] := TransitionClass.alpha; (* k *)
- classification[109] := TransitionClass.alpha; (* l *)
- classification[110] := TransitionClass.alpha; (* m *)
- classification[111] := TransitionClass.alpha; (* n *)
- classification[112] := TransitionClass.alpha; (* o *)
- classification[113] := TransitionClass.alpha; (* p *)
- classification[114] := TransitionClass.alpha; (* q *)
- classification[115] := TransitionClass.alpha; (* r *)
- classification[116] := TransitionClass.alpha; (* s *)
- classification[117] := TransitionClass.alpha; (* t *)
- classification[118] := TransitionClass.alpha; (* u *)
- classification[119] := TransitionClass.alpha; (* v *)
- classification[120] := TransitionClass.alpha; (* w *)
- classification[121] := TransitionClass.x; (* x *)
- classification[122] := TransitionClass.alpha; (* y *)
- classification[123] := TransitionClass.alpha; (* z *)
- classification[124] := TransitionClass.other; (* { *)
- classification[125] := TransitionClass.single; (* | *)
- classification[126] := TransitionClass.other; (* } *)
- classification[127] := TransitionClass.single; (* ~ *)
- classification[128] := TransitionClass.invalid; (* DEL *)
-
- i := 129u;
- while i <= 256u do
- classification[i] := TransitionClass.other;
- i := i + 1u
- end
-end
-
-proc compare_keyword(keyword: String; token_start: BufferPosition; token_end: ^Char): Bool
-var
- result: Bool
- index: Word
- continue: Bool
-begin
- index := 0u;
- result := true;
- continue := (index < keyword.length) & (token_start.iterator <> token_end);
-
- while continue & result do
- result := keyword[index] = token_start.iterator^
- or cast(tolower(cast(keyword[index]: Int)): Char) = token_start.iterator^;
- token_start.iterator := token_start.iterator + 1;
- index := index + 1u;
- continue := (index < keyword.length) & (token_start.iterator <> token_end)
- end;
- result := result & index = keyword.length;
-
- return result & (token_start.iterator = token_end)
-end
-
-(* Reached the end of file. *)
-proc transition_action_eof(lexer: ^Lexer; token: ^LexerToken)
-begin
- token^.kind := LexerKind.unknown
-end
-
-proc increment(position: ^BufferPosition)
-begin
- position^.iterator := position^.iterator + 1
-end
-
-(* Add the character to the token currently read and advance to the next character. *)
-proc transition_action_accumulate(lexer: ^Lexer; token: ^LexerToken)
-begin
- increment(@lexer^.current)
-end
-
-(* The current character is not a part of the token. Finish the token already
- * read. Don't advance to the next character. *)
-proc transition_action_finalize(lexer: ^Lexer; token: ^LexerToken)
-begin
- if lexer^.start.iterator^ = ':' then
- token^.kind := LexerKind.colon
- end;
- if lexer^.start.iterator^ = '>' then
- token^.kind := LexerKind.greater_than
- end;
- if lexer^.start.iterator^ = '<' then
- token^.kind := LexerKind.less_than
- end;
- if lexer^.start.iterator^ = '(' then
- token^.kind := LexerKind.left_paren
- end;
- if lexer^.start.iterator^ = '-' then
- token^.kind := LexerKind.minus
- end;
- if lexer^.start.iterator^ = '.' then
- token^.kind := LexerKind.dot
- end
-end
-
-(* An action for tokens containing multiple characters. *)
-proc transition_action_composite(lexer: ^Lexer; token: ^LexerToken)
-begin
- if lexer^.start.iterator^ = '<' then
- if lexer^.current.iterator^ = '>' then
- token^.kind := LexerKind.not_equal
- end;
- if lexer^.current.iterator^ = '=' then
- token^.kind := LexerKind.less_equal
- end
- end;
- if (lexer^.start.iterator^ = '>') & (lexer^.current.iterator^ = '=') then
- token^.kind := LexerKind.greater_equal
- end;
- if (lexer^.start.iterator^ = ':') & (lexer^.current.iterator^ = '=') then
- token^.kind := LexerKind.assignment
- end;
- if (lexer^.start.iterator^ = '-') & (lexer^.current.iterator^ = '>') then
- token^.kind := LexerKind.arrow
- end;
- increment(@lexer^.current)
-end
-
-(* Skip a space. *)
-proc transition_action_skip(lexer: ^Lexer; token: ^LexerToken)
-begin
- increment(@lexer^.start);
-
- if lexer^.start.iterator^ = '\n' then
- lexer^.start.location.line := lexer^.start.location.line + 1u;
- lexer^.start.location.column := 1u
- end;
- lexer^.current := lexer^.start
-end
-
-(* Delimited string action. *)
-proc transition_action_delimited(lexer: ^Lexer; token: ^LexerToken)
-var
- text_length: Word
-begin
- if lexer^.start.iterator^ = '(' then
- token^.kind := LexerKind.comment
- end;
- if lexer^.start.iterator^ = '"' then
- text_length := cast(lexer^.current.iterator - lexer^.start.iterator + 1: Word);
-
- token^.value.stringKind := String(cast(malloc(text_length): ^Char), text_length);
- memcpy(token^.value.stringKind.ptr, lexer^.start.iterator, text_length);
-
- token^.kind := LexerKind.character
- end;
- if lexer^.start.iterator^ = '\'' then
- text_length := cast(lexer^.current.iterator - lexer^.start.iterator + 1: Word);
-
- token^.value.stringKind := String(cast(malloc(text_length): ^Char), text_length);
- memcpy(token^.value.stringKind.ptr, lexer^.start.iterator, text_length);
-
- token^.kind := LexerKind.string
- end;
- increment(@lexer^.current)
-end
-
-(* Finalize keyword or identifier. *)
-proc transition_action_key_id(lexer: ^Lexer; token: ^LexerToken)
-begin
- token^.kind := LexerKind.identifier;
-
- token^.value.identifierKind[1] := cast(lexer^.current.iterator - lexer^.start.iterator: Char);
- memcpy(@token^.value.identifierKind[2], lexer^.start.iterator, cast(token^.value.identifierKind[1]: Word));
-
- if compare_keyword("program", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._program
- end;
- if compare_keyword("import", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._import
- end;
- if compare_keyword("const", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._const
- end;
- if compare_keyword("var", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._var
- end;
- if compare_keyword("if", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._if
- end;
- if compare_keyword("then", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._then
- end;
- if compare_keyword("elsif", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._elsif
- end;
- if compare_keyword("else", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._else
- end;
- if compare_keyword("while", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._while
- end;
- if compare_keyword("do", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._do
- end;
- if compare_keyword("proc", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._proc
- end;
- if compare_keyword("begin", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._begin
- end;
- if compare_keyword("end", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._end
- end;
- if compare_keyword("type", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._type
- end;
- if compare_keyword("record", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._record
- end;
- if compare_keyword("union", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._union
- end;
- if compare_keyword("NIL", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind.null
- end;
- if compare_keyword("or", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._or
- end;
- if compare_keyword("return", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._return
- end;
- if compare_keyword("defer", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._defer
- end;
- if compare_keyword("TO", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind.to
- end;
- if compare_keyword("CASE", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._case
- end;
- if compare_keyword("OF", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._of
- end;
- if compare_keyword("module", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._module
- end;
- if compare_keyword("xor", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind._xor
- end;
- if compare_keyword("TRUE", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind.boolean;
- token^.value.booleanKind := true
- end;
- if compare_keyword("FALSE", lexer^.start, lexer^.current.iterator) then
- token^.kind := LexerKind.boolean;
- token^.value.booleanKind := false
- end
-end
-
-(* Action for tokens containing only one character. The character cannot be
- * followed by other characters forming a composite token. *)
-proc transition_action_single(lexer: ^Lexer; token: ^LexerToken)
-begin
- if lexer^.current.iterator^ = '&' then
- token^.kind := LexerKind.and
- end;
- if lexer^.current.iterator^ = ';' then
- token^.kind := LexerKind.semicolon
- end;
- if lexer^.current.iterator^ = ',' then
- token^.kind := LexerKind.comma
- end;
- if lexer^.current.iterator^ = '~' then
- token^.kind := LexerKind.not
- end;
- if lexer^.current.iterator^ = ')' then
- token^.kind := LexerKind.right_paren
- end;
- if lexer^.current.iterator^ = '[' then
- token^.kind := LexerKind.left_square
- end;
- if lexer^.current.iterator^ = ']' then
- token^.kind := LexerKind.right_square
- end;
- if lexer^.current.iterator^ = '^' then
- token^.kind := LexerKind.hat
- end;
- if lexer^.current.iterator^ = '=' then
- token^.kind := LexerKind.equal
- end;
- if lexer^.current.iterator^ = '+' then
- token^.kind := LexerKind.plus
- end;
- if lexer^.current.iterator^ = '*' then
- token^.kind := LexerKind.multiplication
- end;
- if lexer^.current.iterator^ = '/' then
- token^.kind := LexerKind.division
- end;
- if lexer^.current.iterator^ = '%' then
- token^.kind := LexerKind.remainder
- end;
- if lexer^.current.iterator^ = '@' then
- token^.kind := LexerKind.at
- end;
- if lexer^.current.iterator^ = '|' then
- token^.kind := LexerKind.pipe
- end;
- increment(@lexer^.current)
-end
-
-(* Handle an integer literal. *)
-proc transition_action_integer(lexer: ^Lexer; token: ^LexerToken)
-var
- buffer: String
- integer_length: Word
- found: Bool
-begin
- token^.kind := LexerKind.integer;
-
- integer_length := cast(lexer^.current.iterator - lexer^.start.iterator: Word);
- memset(cast(token^.value.identifierKind.ptr: Pointer), 0, #size(Identifier));
- memcpy(@token^.value.identifierKind[1], lexer^.start.iterator, integer_length);
-
- token^.value.identifierKind[cast(token^.value.identifierKind[1]: Int) + 2] := '\0';
- token^.value.integerKind := atoi(@token^.value.identifierKind[2])
-end
-
-proc set_default_transition(current_state: TransitionState; default_action: TransitionAction; next_state: TransitionState): Int
-var
- default_transition: Transition
- state_index: Int
-begin
- default_transition.action := default_action;
- default_transition.next_state := next_state;
- state_index := cast(current_state: Int) + 1;
-
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.digit: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.space: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.colon: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.equals: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.left_paren: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.right_paren: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.single: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.hex: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.zero: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.x: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.eof: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.dot: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.minus: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.single_quote: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.double_quote: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.greater: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.less: Int) + 1] := default_transition;
- transitions[state_index][cast(TransitionClass.other: Int) + 1] := default_transition;
-
- return state_index
-end
-
-(*
- * The transition table describes transitions from one state to another, given
- * a symbol (character class).
- *
- * The table has m rows and n columns, where m is the amount of states and n is
- * the amount of classes. So given the current state and a classified character
- * the table can be used to look up the next state.
- *
- * Each cell is a word long.
- * - The least significant byte of the word is a row number (beginning with 0).
- * It specifies the target state. "ff" means that this is an end state and no
- * transition is possible.
- * - The next byte is the action that should be performed when transitioning.
- * For the meaning of actions see labels in the lex_next function, which
- * handles each action.
- *)
-proc initialize_transitions()
-var
- state_index: Int
-begin
- (* Start state. *)
- state_index := cast(TransitionState.start: Int) + 1;
-
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.decimal;
-
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.space: Int) + 1].action := transition_action_skip;
- transitions[state_index][cast(TransitionClass.space: Int) + 1].next_state := TransitionState.start;
-
- transitions[state_index][cast(TransitionClass.colon: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.colon: Int) + 1].next_state := TransitionState.colon;
-
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_single;
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.left_paren: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.left_paren: Int) + 1].next_state := TransitionState.left_paren;
-
- transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].action := transition_action_single;
- transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_single;
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.single: Int) + 1].action := transition_action_single;
- transitions[state_index][cast(TransitionClass.single: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.leading_zero;
-
- transitions[state_index][cast(TransitionClass.x: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := transition_action_eof;
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.dot: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.dot: Int) + 1].next_state := TransitionState.dot;
-
- transitions[state_index][cast(TransitionClass.minus: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.minus: Int) + 1].next_state := TransitionState.minus;
-
- transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].next_state := TransitionState.character;
-
- transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].next_state := TransitionState.string;
-
- transitions[state_index][cast(TransitionClass.greater: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.greater: Int) + 1].next_state := TransitionState.greater;
-
- transitions[state_index][cast(TransitionClass.less: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.less: Int) + 1].next_state := TransitionState.less;
-
- transitions[state_index][cast(TransitionClass.other: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.other: Int) + 1].next_state := TransitionState.finish;
-
- (* Colon state. *)
- state_index := set_default_transition(TransitionState.colon, transition_action_finalize, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_composite;
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish;
-
- (* Identifier state. *)
- state_index := set_default_transition(TransitionState.identifier, transition_action_key_id, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.identifier;
-
- transitions[state_index][cast(TransitionClass.x: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.identifier;
-
- (* Decimal state. *)
- state_index := set_default_transition(TransitionState.decimal, transition_action_integer, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.decimal;
-
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.decimal_suffix;
-
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.decimal_suffix;
-
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.decimal;
-
- transitions[state_index][cast(TransitionClass.x: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.decimal_suffix;
-
- (* Greater state. *)
- state_index := set_default_transition(TransitionState.greater, transition_action_finalize, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_composite;
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish;
-
- (* Minus state. *)
- state_index := set_default_transition(TransitionState.minus, transition_action_finalize, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.greater: Int) + 1].action := transition_action_composite;
- transitions[state_index][cast(TransitionClass.greater: Int) + 1].next_state := TransitionState.finish;
-
- (* Left paren state. *)
- state_index := set_default_transition(TransitionState.left_paren, transition_action_finalize, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.comment;
-
- (* Less state. *)
- state_index := set_default_transition(TransitionState.less, transition_action_finalize, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_composite;
- transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.greater: Int) + 1].action := transition_action_composite;
- transitions[state_index][cast(TransitionClass.greater: Int) + 1].next_state := TransitionState.finish;
-
- (* Hexadecimal after 0x. *)
- state_index := set_default_transition(TransitionState.dot, transition_action_finalize, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.dot: Int) + 1].action := transition_action_composite;
- transitions[state_index][cast(TransitionClass.dot: Int) + 1].next_state := TransitionState.finish;
-
- (* Comment. *)
- state_index := set_default_transition(TransitionState.comment, transition_action_accumulate, TransitionState.comment);
-
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.closing_comment;
-
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish;
-
- (* Closing comment. *)
- state_index := set_default_transition(TransitionState.closing_comment, transition_action_accumulate, TransitionState.comment);
-
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].action := transition_action_delimited;
- transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_accumulate;
- transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.closing_comment;
-
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish;
-
- (* Character. *)
- state_index := set_default_transition(TransitionState.character, transition_action_accumulate, TransitionState.character);
-
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].action := transition_action_delimited;
- transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].next_state := TransitionState.finish;
-
- (* String. *)
- state_index := set_default_transition(TransitionState.string, transition_action_accumulate, TransitionState.string);
-
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].action := transition_action_delimited;
- transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].next_state := TransitionState.finish;
-
- (* Leading zero. *)
- state_index := set_default_transition(TransitionState.leading_zero, transition_action_integer, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.x: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.finish;
-
- (* Digit with a character suffix. *)
- state_index := set_default_transition(TransitionState.decimal_suffix, transition_action_integer, TransitionState.finish);
-
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.finish;
-
- transitions[state_index][cast(TransitionClass.x: Int) + 1].action := nil;
- transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.finish
-end
-
-proc lexer_make*(lexer: ^Lexer; input: ^FILE)
-begin
- lexer^.input := input;
- lexer^.length := 0u;
-
- lexer^.buffer := cast(malloc(CHUNK_SIZE): ^Char);
- memset(cast(lexer^.buffer: Pointer), 0, CHUNK_SIZE);
- lexer^.size := CHUNK_SIZE
-end
-
-(* Returns the last read token. *)
-proc lexer_current*(lexer: ^Lexer): LexerToken
-var
- current_class: TransitionClass
- current_state: TransitionState
- current_transition: Transition
- result: LexerToken
- index1: Word
- index2: Word
-begin
- lexer^.current := lexer^.start;
- current_state := TransitionState.start;
-
- while current_state <> TransitionState.finish do
- index1 := cast(lexer^.current.iterator^: Word) + 1u;
- current_class := classification[index1];
-
- index1 := cast(current_state: Word) + 1u;
- index2 := cast(current_class: Word) + 1u;
-
- current_transition := transitions[index1][index2];
- if current_transition.action <> nil then
- current_transition.action(lexer, @result)
- end;
- current_state := current_transition.next_state
- end;
- result.start_location := lexer^.start.location;
- result.end_location := lexer^.current.location;
-
- return result
-end
-
-(* Read and return the next token. *)
-proc lexer_lex*(lexer: ^Lexer): LexerToken
-var
- result: LexerToken
-begin
- if lexer^.length = 0u then
- lexer^.length := fread(cast(lexer^.buffer: Pointer), CHUNK_SIZE, 1u, lexer^.input);
- lexer^.current.location.column := 1u;
- lexer^.current.location.line := 1u;
- lexer^.current.iterator := lexer^.buffer
- end;
- lexer^.start := lexer^.current;
-
- result := lexer_current(lexer);
- return result
-end
-
-proc lexer_destroy*(lexer: ^Lexer)
-begin
- free(cast(lexer^.buffer: Pointer))
-end
-
-proc lexer_initialize()
-begin
- initialize_classification();
- initialize_transitions()
-end
+ shift_right,
+ shift_left,
+ pipe,
+ _case,
+ _do,
+ _of,
+ eof
+ )
+ (**
+ * Classification table assigns each possible character to a group (class). All
+ * characters of the same group a handled equivalently.
+ *)
+ ElnaLexerClass = (
+ invalid,
+ digit,
+ alpha,
+ space,
+ colon,
+ equals,
+ left_paren,
+ right_paren,
+ asterisk,
+ backslash,
+ single,
+ hex,
+ zero,
+ x,
+ eof,
+ dot,
+ minus,
+ single_quote,
+ double_quote,
+ greater,
+ less,
+ other,
+ number_sign
+ )
end.
diff --git a/source/main.elna b/source/main.elna
index aeaba2f..781d681 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -24,18 +24,9 @@ type
advance: proc(stream: Pointer);
head: proc(stream: Pointer): Char
end
- Token* = record
- kind: LexerKind;
- value: union
- int_value: Int;
- string: String;
- boolean_value: Bool;
- char_value: Char
- end
- end
Tokenizer* = record
length: Word;
- data: ^Token
+ data: ^^ElnaLexerToken
end
(*
@@ -57,7 +48,7 @@ proc string_dup(origin: String): String
var
copy: ^Char
begin
- copy := cast(malloc(origin.length): ^Char);
+ copy := malloc(origin.length);
strncpy(copy, origin.ptr, origin.length);
return String(copy, origin.length)
@@ -326,92 +317,109 @@ begin
end
(* Categorize an identifier. *)
-proc lexer_categorize(token_content: String): Token
+proc lexer_categorize(token_content: String): ^ElnaLexerToken
var
- current_token: Token
+ current_token: ^ElnaLexerToken
begin
if token_content = "if" then
- current_token.kind := LexerKind._if
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._if
elsif token_content = "then" then
- current_token.kind := LexerKind._then
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._then
elsif token_content = "else" then
- current_token.kind := LexerKind._else
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._else
elsif token_content = "elsif" then
- current_token.kind := LexerKind._elsif
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._elsif
elsif token_content = "while" then
- current_token.kind := LexerKind._while
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._while
elsif token_content = "do" then
- current_token.kind := LexerKind._do
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._do
elsif token_content = "proc" then
- current_token.kind := LexerKind._proc
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._proc
elsif token_content = "begin" then
- current_token.kind := LexerKind._begin
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._begin
elsif token_content = "end" then
- current_token.kind := LexerKind._end
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._end
elsif token_content = "extern" then
- current_token.kind := LexerKind._extern
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._extern
elsif token_content = "const" then
- current_token.kind := LexerKind._const
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._const
elsif token_content = "var" then
- current_token.kind := LexerKind._var
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._var
elsif token_content = "case" then
- current_token.kind := LexerKind._case
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._case
elsif token_content = "of" then
- current_token.kind := LexerKind._of
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._of
elsif token_content = "type" then
- current_token.kind := LexerKind._type
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._type
elsif token_content = "record" then
- current_token.kind := LexerKind._record
- elsif token_content = "union" then
- current_token.kind := LexerKind._union
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._record
elsif token_content = "true" then
- current_token.kind := LexerKind.boolean;
- current_token.value.boolean_value := true
+ current_token := malloc(#size(ElnaLexerBooleanToken));
+ current_token^.kind := ElnaLexerKind.boolean;
+ cast(current_token: ^ElnaLexerBooleanToken)^.value := true
elsif token_content = "false" then
- current_token.kind := LexerKind.boolean;
- current_token.value.boolean_value := false
+ current_token := malloc(#size(ElnaLexerBooleanToken));
+ current_token^.kind := ElnaLexerKind.boolean;
+ cast(current_token: ^ElnaLexerBooleanToken)^.value := false
elsif token_content = "nil" then
- current_token.kind := LexerKind.null
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.null
elsif token_content = "or" then
- current_token.kind := LexerKind._or
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._or
elsif token_content = "return" then
- current_token.kind := LexerKind._return
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._return
elsif token_content = "cast" then
- current_token.kind := LexerKind._cast
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._cast
elsif token_content = "defer" then
- current_token.kind := LexerKind._defer
- elsif token_content = "program" then
- current_token.kind := LexerKind._program
- elsif token_content = "module" then
- current_token.kind := LexerKind._module
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._defer
elsif token_content = "import" then
- current_token.kind := LexerKind._import
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind._import
else
- current_token.kind := LexerKind.identifier;
- current_token.value.string := string_dup(token_content)
+ current_token := malloc(#size(ElnaLexerStringToken));
+ current_token^.kind := ElnaLexerKind.identifier;
+ cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(token_content)
end;
return current_token
end
-proc lexer_add_token(lexer: ^Tokenizer; token: Token)
+proc lexer_add_token(lexer: ^Tokenizer; token: ^ElnaLexerToken)
var
new_length: Word
begin
new_length := lexer^.length + 1u;
- lexer^.data := cast(reallocarray(cast(lexer^.data: Pointer), new_length, #size(Token)): ^Token);
+ lexer^.data := cast(reallocarray(cast(lexer^.data: Pointer), new_length, #size(Pointer)): ^^ElnaLexerToken);
(lexer^.data + lexer^.length)^ := token;
lexer^.length := new_length
end
(* Read the next token from the input. *)
-proc lexer_next(source_code: SourceCode; token_buffer: ^StringBuffer): Token
+proc lexer_next(source_code: SourceCode; token_buffer: ^StringBuffer): ^ElnaLexerToken
var
- current_token: Token
+ current_token: ^ElnaLexerToken := nil
first_char: Char
begin
- current_token.kind := LexerKind.unknown;
-
first_char := source_code_head(source_code);
if isalpha(cast(first_char: Int)) <> 0 or first_char = '_' then
@@ -421,158 +429,170 @@ begin
source_code_advance(@source_code);
lexer_identifier(@source_code, token_buffer);
- current_token.kind := LexerKind.trait;
- current_token.value.string := string_dup(string_buffer_clear(token_buffer))
+ current_token := malloc(#size(ElnaLexerStringToken));
+ current_token^.kind := ElnaLexerKind.trait;
+ cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(string_buffer_clear(token_buffer))
elsif isdigit(cast(first_char: Int)) <> 0 then
- lexer_number(@source_code, @current_token.value.int_value);
+ current_token := malloc(#size(ElnaLexerIntegerToken));
+ lexer_number(@source_code, @cast(current_token: ^ElnaLexerIntegerToken)^.value);
if source_code_expect(@source_code, 'u') then
- current_token.kind := LexerKind.word;
- source_code_advance(@source_code)
+ current_token^.kind := ElnaLexerKind.word;
+ source_code_advance(@source_code)
else
- current_token.kind := LexerKind.integer
+ current_token^.kind := ElnaLexerKind.integer
end
elsif first_char = '(' then
source_code_advance(@source_code);
if source_code_empty(@source_code) then
- current_token.kind := LexerKind.left_paren
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.left_paren
elsif source_code_head(source_code) = '*' then
source_code_advance(@source_code);
if lexer_comment(@source_code, token_buffer) then
- current_token.value.string := string_dup(string_buffer_clear(token_buffer));
- current_token.kind := LexerKind.comment
- else
- current_token.kind := LexerKind.unknown
+ current_token := malloc(#size(ElnaLexerStringToken));
+ cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(string_buffer_clear(token_buffer));
+ current_token^.kind := ElnaLexerKind.comment
end
else
- current_token.kind := LexerKind.left_paren
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.left_paren
end
elsif first_char = ')' then
- current_token.kind := LexerKind.right_paren;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.right_paren;
source_code_advance(@source_code)
elsif first_char = '\'' then
source_code_advance(@source_code);
- if lexer_character(@source_code, @current_token.value.char_value) & source_code_expect(@source_code, '\'') then
- current_token.kind := LexerKind.character;
+ if lexer_character(@source_code, @first_char) & source_code_expect(@source_code, '\'') then
+ current_token := malloc(#size(ElnaLexerCharacterToken));
+ current_token^.kind := ElnaLexerKind.character;
+ cast(current_token: ^ElnaLexerCharacterToken)^.value := first_char;
source_code_advance(@source_code)
- else
- current_token.kind := LexerKind.unknown
end
elsif first_char = '"' then
source_code_advance(@source_code);
if lexer_string(@source_code, token_buffer) then
- current_token.kind := LexerKind.string;
- current_token.value.string := string_dup(string_buffer_clear(token_buffer))
- else
- current_token.kind := LexerKind.unknown
+ current_token := malloc(#size(ElnaLexerStringToken));
+ current_token^.kind := ElnaLexerKind.string;
+ cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(string_buffer_clear(token_buffer))
end
elsif first_char = '[' then
- current_token.kind := LexerKind.left_square;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.left_square;
source_code_advance(@source_code)
elsif first_char = ']' then
- current_token.kind := LexerKind.right_square;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.right_square;
source_code_advance(@source_code)
elsif first_char = '>' then
source_code_advance(@source_code);
+ current_token := malloc(#size(ElnaLexerToken));
if source_code_empty(@source_code) then
- current_token.kind := LexerKind.greater_than
+ current_token^.kind := ElnaLexerKind.greater_than
elsif source_code_head(source_code) = '=' then
- current_token.kind := LexerKind.greater_equal;
+ current_token^.kind := ElnaLexerKind.greater_equal;
source_code_advance(@source_code)
elsif source_code_head(source_code) = '>' then
- current_token.kind := LexerKind.shift_right;
+ current_token^.kind := ElnaLexerKind.shift_right;
source_code_advance(@source_code)
else
- current_token.kind := LexerKind.greater_than
+ current_token^.kind := ElnaLexerKind.greater_than
end
elsif first_char = '<' then
source_code_advance(@source_code);
+ current_token := malloc(#size(ElnaLexerToken));
if source_code_empty(@source_code) then
- current_token.kind := LexerKind.less_than
+ current_token^.kind := ElnaLexerKind.less_than
elsif source_code_head(source_code) = '=' then
- current_token.kind := LexerKind.less_equal;
+ current_token^.kind := ElnaLexerKind.less_equal;
source_code_advance(@source_code)
elsif source_code_head(source_code) = '<' then
- current_token.kind := LexerKind.shift_left;
+ current_token^.kind := ElnaLexerKind.shift_left;
source_code_advance(@source_code)
elsif source_code_head(source_code) = '>' then
- current_token.kind := LexerKind.not_equal;
+ current_token^.kind := ElnaLexerKind.not_equal;
source_code_advance(@source_code)
else
- current_token.kind := LexerKind.less_than
+ current_token^.kind := ElnaLexerKind.less_than
end
elsif first_char = '=' then
- current_token.kind := LexerKind.equal;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.equals;
source_code_advance(@source_code)
elsif first_char = ';' then
- current_token.kind := LexerKind.semicolon;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.semicolon;
source_code_advance(@source_code)
elsif first_char = '.' then
- current_token.kind := LexerKind.dot;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.dot;
source_code_advance(@source_code)
elsif first_char = ',' then
- current_token.kind := LexerKind.comma;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.comma;
source_code_advance(@source_code)
elsif first_char = '+' then
- current_token.kind := LexerKind.plus;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.plus;
source_code_advance(@source_code)
elsif first_char = '-' then
source_code_advance(@source_code);
-
- if source_code_empty(@source_code) then
- current_token.kind := LexerKind.minus
- elsif source_code_head(source_code) = '>' then
- current_token.kind := LexerKind.arrow;
- source_code_advance(@source_code)
- else
- current_token.kind := LexerKind.minus
- end
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.minus
elsif first_char = '*' then
- current_token.kind := LexerKind.multiplication;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.multiplication;
source_code_advance(@source_code)
elsif first_char = '/' then
- current_token.kind := LexerKind.division;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.division;
source_code_advance(@source_code)
elsif first_char = '%' then
- current_token.kind := LexerKind.remainder;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.remainder;
source_code_advance(@source_code)
elsif first_char = ':' then
source_code_advance(@source_code);
+ current_token := malloc(#size(ElnaLexerToken));
if source_code_empty(@source_code) then
- current_token.kind := LexerKind.colon
+ current_token^.kind := ElnaLexerKind.colon
elsif source_code_head(source_code) = '=' then
- current_token.kind := LexerKind.assignment;
+ current_token^.kind := ElnaLexerKind.assignment;
source_code_advance(@source_code)
else
- current_token.kind := LexerKind.colon
+ current_token^.kind := ElnaLexerKind.colon
end
elsif first_char = '^' then
- current_token.kind := LexerKind.hat;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.hat;
source_code_advance(@source_code)
elsif first_char = '@' then
- current_token.kind := LexerKind.at;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.at;
source_code_advance(@source_code)
elsif first_char = '!' then
- current_token.kind := LexerKind.exclamation;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.exclamation;
source_code_advance(@source_code)
elsif first_char = '&' then
- current_token.kind := LexerKind.and;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.and;
source_code_advance(@source_code)
elsif first_char = '~' then
- current_token.kind := LexerKind.not;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.not;
source_code_advance(@source_code)
elsif first_char = '|' then
- current_token.kind := LexerKind.pipe;
- source_code_advance(@source_code)
- else
- current_token.kind := LexerKind.unknown;
+ current_token := malloc(#size(ElnaLexerToken));
+ current_token^.kind := ElnaLexerKind.pipe;
source_code_advance(@source_code)
end;
@@ -582,7 +602,7 @@ end
(* Split the source text into tokens. *)
proc lexer_text(source_code: SourceCode): Tokenizer
var
- current_token: Token
+ current_token: ^ElnaLexerToken
token_buffer: StringBuffer
lexer: Tokenizer
begin
@@ -594,7 +614,7 @@ begin
while ~source_code_empty(@source_code) do
current_token := lexer_next(source_code, @token_buffer);
- if current_token.kind <> LexerKind.unknown then
+ if current_token <> nil then
lexer_add_token(@lexer, current_token);
lexer_spaces(@source_code)
else
@@ -611,151 +631,141 @@ end
Parser.
*)
-proc parse(tokens: ^Token; tokens_size: Word)
+proc parse(tokens: ^^ElnaLexerToken; tokens_size: Word)
var
- current_token: ^Token
+ current_token: ^ElnaLexerToken
i: Word := 0u
begin
while i < tokens_size do
- current_token := tokens + i;
+ current_token := (tokens + i)^;
case current_token^.kind of
- LexerKind._if:
+ ElnaLexerKind._if:
write_s("IF")
- | LexerKind._then:
+ | ElnaLexerKind._then:
write_s("THEN")
- | LexerKind._else:
+ | ElnaLexerKind._else:
write_s("ELSE")
- | LexerKind._elsif:
+ | ElnaLexerKind._elsif:
write_s("ELSIF")
- | LexerKind._while:
+ | ElnaLexerKind._while:
write_s("WHILE")
- | LexerKind._do:
+ | ElnaLexerKind._do:
write_s("DO")
- | LexerKind._proc:
+ | ElnaLexerKind._proc:
write_s("PROC")
- | LexerKind._begin:
+ | ElnaLexerKind._begin:
write_s("BEGIN")
- | LexerKind._end:
+ | ElnaLexerKind._end:
write_s("END")
- | LexerKind._extern:
+ | ElnaLexerKind._extern:
write_s("EXTERN")
- | LexerKind._const:
+ | ElnaLexerKind._const:
write_s("CONST")
- | LexerKind._var:
+ | ElnaLexerKind._var:
write_s("VAR")
- | LexerKind._case:
+ | ElnaLexerKind._case:
write_s("CASE")
- | LexerKind._of:
+ | ElnaLexerKind._of:
write_s("OF")
- | LexerKind._type:
+ | ElnaLexerKind._type:
write_s("TYPE")
- | LexerKind._record:
+ | ElnaLexerKind._record:
write_s("RECORD")
- | LexerKind._union:
- write_s("UNION")
- | LexerKind.pipe:
+ | ElnaLexerKind.pipe:
write_s("|")
- | LexerKind.to:
- write_s("TO")
- | LexerKind.boolean:
+ | ElnaLexerKind.boolean:
write_s("BOOLEAN<");
- write_b(current_token^.value.boolean_value);
+ write_b(cast(current_token: ^ElnaLexerBooleanToken)^.value);
write_c('>')
- | LexerKind.null:
+ | ElnaLexerKind.null:
write_s("NIL")
- | LexerKind.and:
+ | ElnaLexerKind.and:
write_s("&")
- | LexerKind._or:
+ | ElnaLexerKind._or:
write_s("OR")
- | LexerKind.not:
+ | ElnaLexerKind.not:
write_s("~")
- | LexerKind._return:
+ | ElnaLexerKind._return:
write_s("RETURN")
- | LexerKind._cast:
+ | ElnaLexerKind._cast:
write_s("CAST")
- | LexerKind.shift_left:
+ | ElnaLexerKind.shift_left:
write_s("<<")
- | LexerKind.shift_right:
+ | ElnaLexerKind.shift_right:
write_s(">>")
- | LexerKind.identifier:
+ | ElnaLexerKind.identifier:
write_c('<');
- write_s(current_token^.value.string);
+ write_s(cast(current_token: ^ElnaLexerStringToken)^.value);
write_c('>')
- | LexerKind.trait:
+ | ElnaLexerKind.trait:
write_c('#');
- write_s(current_token^.value.string)
- | LexerKind.left_paren:
+ write_s(cast(current_token: ^ElnaLexerStringToken)^.value)
+ | ElnaLexerKind.left_paren:
write_s("(")
- | LexerKind.right_paren:
+ | ElnaLexerKind.right_paren:
write_s(")")
- | LexerKind.left_square:
+ | ElnaLexerKind.left_square:
write_s("[")
- | LexerKind.right_square:
+ | ElnaLexerKind.right_square:
write_s("]")
- | LexerKind.greater_equal:
+ | ElnaLexerKind.greater_equal:
write_s(">=")
- | LexerKind.less_equal:
+ | ElnaLexerKind.less_equal:
write_s("<=")
- | LexerKind.greater_than:
+ | ElnaLexerKind.greater_than:
write_s(">")
- | LexerKind.less_than:
+ | ElnaLexerKind.less_than:
write_s("<")
- | LexerKind.equal:
+ | ElnaLexerKind.equals:
write_s("=")
- | LexerKind.not_equal:
+ | ElnaLexerKind.not_equal:
write_s("<>")
- | LexerKind.semicolon:
+ | ElnaLexerKind.semicolon:
write_c(';')
- | LexerKind.dot:
+ | ElnaLexerKind.dot:
write_c('.')
- | LexerKind.comma:
+ | ElnaLexerKind.comma:
write_c(',')
- | LexerKind.plus:
+ | ElnaLexerKind.plus:
write_c('+')
- | LexerKind.minus:
+ | ElnaLexerKind.minus:
write_c('-')
- | LexerKind.multiplication:
+ | ElnaLexerKind.multiplication:
write_c('*')
- | LexerKind.division:
+ | ElnaLexerKind.division:
write_c('/')
- | LexerKind.remainder:
+ | ElnaLexerKind.remainder:
write_c('%')
- | LexerKind.assignment:
+ | ElnaLexerKind.assignment:
write_s(":=")
- | LexerKind.colon:
+ | ElnaLexerKind.colon:
write_c(':')
- | LexerKind.hat:
+ | ElnaLexerKind.hat:
write_c('^')
- | LexerKind.at:
+ | ElnaLexerKind.at:
write_c('@')
- | LexerKind.comment:
+ | ElnaLexerKind.comment:
write_s("(* COMMENT *)")
- | LexerKind.integer:
+ | ElnaLexerKind.integer:
write_c('<');
- write_i(current_token^.value.int_value);
+ write_i(cast(current_token: ^ElnaLexerIntegerToken)^.value);
write_c('>')
- | LexerKind.word:
+ | ElnaLexerKind.word:
write_c('<');
- write_i(current_token^.value.int_value);
+ write_i(cast(current_token: ^ElnaLexerIntegerToken)^.value);
write_s("u>")
- | LexerKind.character:
+ | ElnaLexerKind.character:
write_c('<');
- write_i(cast(current_token^.value.char_value: Int));
+ write_i(cast(cast(current_token: ^ElnaLexerCharacterToken)^.value: Int));
write_s("c>")
- | LexerKind.string:
+ | ElnaLexerKind.string:
write_s("\"...\"")
- | LexerKind._defer:
+ | ElnaLexerKind._defer:
write_s("DEFER")
- | LexerKind.exclamation:
+ | ElnaLexerKind.exclamation:
write_c('!')
- | LexerKind.arrow:
- write_s("->")
- | LexerKind._program:
- write_s("PROGRAM")
- | LexerKind._module:
- write_s("MODULE")
- | LexerKind._import:
+ | ElnaLexerKind._import:
write_s("IMPORT")
else
write_s("UNKNOWN<");
@@ -790,7 +800,7 @@ end
proc process(argc: Int; argv: ^^Char): Int
var
- tokens: ^Token
+ tokens: ^ElnaLexerToken
tokens_size: Word
source_code: SourceCode
command_line: ^CommandLine
@@ -817,7 +827,7 @@ begin
end;
source_code.position := TextLocation(1u, 1u);
- source_code.input := cast(source_file: Pointer);
+ source_code.input := source_file;
source_code.empty := source_file_empty;
source_code.head := source_file_head;
source_code.advance := source_file_advance;