diff options
| author | Eugen Wissner <belka@caraus.de> | 2026-07-12 13:23:54 +0200 |
|---|---|---|
| committer | Eugen Wissner <belka@caraus.de> | 2026-07-12 13:23:54 +0200 |
| commit | 768537d6e4b82a19e77b7e44234309ecaf4d65d2 (patch) | |
| tree | ab22613a16c3a914b16d00a5f988f51d1eadac0d /source | |
| parent | 14d4977e2ab2409bb7344395ca01d19e49f130f1 (diff) | |
| download | elna-768537d6e4b82a19e77b7e44234309ecaf4d65d2.tar.gz | |
Remove unions
Diffstat (limited to 'source')
| -rw-r--r-- | source/cstdio.elna | 12 | ||||
| -rw-r--r-- | source/cstdlib.elna | 12 | ||||
| -rw-r--r-- | source/cstring.elna | 17 | ||||
| -rw-r--r-- | source/lexer.elna | 1050 | ||||
| -rw-r--r-- | source/main.elna | 400 |
5 files changed, 361 insertions, 1130 deletions
diff --git a/source/cstdio.elna b/source/cstdio.elna index 95283a1..346d931 100644 --- a/source/cstdio.elna +++ b/source/cstdio.elna @@ -34,6 +34,9 @@ extern proc fwrite*(ptr: Pointer; size: Word; nitems: Word; stream: ^FILE): Word extern +proc fputc(c: Int; stream: ^FILE): Word +extern + proc perror(s: ^Char) extern @@ -43,4 +46,13 @@ extern proc putchar(c: Int): Int extern +proc sprintf(str: Pointer; format: ^Char; number: Word): Int +extern + +proc fprintf(stream: Pointer; format: ^Char; number: Word): Int +extern + +proc fdopen(fildes: Int; mode: ^Char): Pointer +extern + end. diff --git a/source/cstdlib.elna b/source/cstdlib.elna index 696e5dd..32669c1 100644 --- a/source/cstdlib.elna +++ b/source/cstdlib.elna @@ -2,22 +2,22 @@ v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. *) -proc malloc(size: Word): Pointer +proc malloc*(size: Word): Pointer extern -proc free(ptr: Pointer) +proc free*(ptr: Pointer) extern -proc calloc(nmemb: Word; size: Word): Pointer +proc calloc*(nmemb: Word; size: Word): Pointer extern -proc realloc(ptr: Pointer; size: Word): Pointer +proc realloc*(ptr: Pointer; size: Word): Pointer extern -proc atoi(str: ^Char): Int +proc atoi*(str: ^Char): Int extern -proc exit(code: Int): ! +proc exit*(code: Int): ! extern end. diff --git a/source/cstring.elna b/source/cstring.elna index ec5cd7b..e21056a 100644 --- a/source/cstring.elna +++ b/source/cstring.elna @@ -2,25 +2,28 @@ v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. *) -proc memset(ptr: Pointer; c: Int; n: Word): ^Char +proc memset*(ptr: Pointer; c: Int; n: Word): ^Char extern -proc memcpy(dst: Pointer; src: Pointer; n: Word) +proc memcpy*(dst, src: Pointer; n: Word) extern -proc strcmp(s1: ^Char; s2: ^Char): Int +proc memcmp*(s1, s2: Pointer; n: Word): Int extern -proc strncmp(s1: ^Char; s2: ^Char; n: Word): Int +proc strcmp*(s1: ^Char; s2: ^Char): Int extern -proc strncpy(dst: ^Char; src: ^Char; dsize: Word): ^Char +proc strncmp*(s1: ^Char; s2: ^Char; n: Word): Int extern -proc strcpy(dst: ^Char; src: ^Char): ^Char +proc strncpy*(dst: ^Char; src: ^Char; dsize: Word): ^Char extern -proc strlen(ptr: ^Char): Word +proc strcpy*(dst: ^Char; src: ^Char): ^Char +extern + +proc strlen*(ptr: ^Char): Word extern end. diff --git a/source/lexer.elna b/source/lexer.elna index bb8c8ef..3b015ad 100644 --- a/source/lexer.elna +++ b/source/lexer.elna @@ -2,77 +2,53 @@ v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. *) -import cstdio, cstring, cctype, cstdlib, common +import cstdio, common const CHUNK_SIZE := 85536u type - (* - * Classification table assigns each possible character to a group (class). All - * characters of the same group are handled equivalently. - * - * Classification: - *) - TransitionClass = ( - invalid, - digit, - alpha, - space, - colon, - equals, - left_paren, - right_paren, - asterisk, - underscore, - single, - hex, - zero, - x, - eof, - dot, - minus, - single_quote, - double_quote, - greater, - less, - other - ) - TransitionState = ( - start, - colon, - identifier, - decimal, - greater, - minus, - left_paren, - less, - dot, - comment, - closing_comment, - character, - string, - leading_zero, - decimal_suffix, - finish - ) - LexerToken = record - kind: LexerKind; - value: union - booleanKind: Bool; - identifierKind: Identifier; - integerKind: Int; - stringKind: String - end; - start_location: TextLocation; - end_location: TextLocation + ElnaLexerState = ( + start, + colon, + identifier, + decimal, + leading_zero, + greater, + minus, + left_paren, + less, + dot, + comment, + closing_comment, + character, + character_escape, + string, + string_escape, + number_sign, + trait, + finish + ) + ElnaLexerToken* = record + kind: ElnaLexerKind end - TransitionAction = proc(lexer: ^Lexer; token: ^LexerToken) - Transition = record + ElnaLexerBooleanToken* = record(ElnaLexerToken) + value: Bool + end + ElnaLexerCharacterToken* = record(ElnaLexerToken) + value: Char + end + ElnaLexerStringToken* = record(ElnaLexerToken) + value: String + end + ElnaLexerIntegerToken* = record(ElnaLexerToken) + value: Int + end + TransitionAction = proc(lexer: ^Lexer; token: ^ElnaLexerToken) + ElnaLexerTransition = record action: TransitionAction; - next_state: TransitionState + next_state: ElnaLexerState end - TransitionClasses = [22]Transition BufferPosition* = record iterator: ^Char; @@ -86,866 +62,96 @@ type start: BufferPosition; current: BufferPosition end - LexerKind* = ( - unknown, - identifier, - _if, - _then, - _else, - _elsif, - _while, - _do, - _proc, - _begin, - _end, - _extern, - _const, - _var, - _case, - _of, - _type, - _record, - _union, - pipe, - to, - boolean, - null, - and, - _or, - _xor, - not, - _return, - _cast, - shift_left, - shift_right, - left_paren, - right_paren, - left_square, - right_square, - greater_equal, - less_equal, - greater_than, - less_than, - not_equal, - equal, - semicolon, - dot, - comma, - plus, - minus, - multiplication, - division, - remainder, - assignment, - colon, - hat, - at, - comment, - integer, - word, - character, - string, + ElnaLexerKind* = ( + identifier, + _const, + _var, + _proc, + _type, + _begin, + _end, + _if, + _then, + _else, + _elsif, + _extern, + _record, + boolean, + null, + and, + _or, + _xor, + not, + _return, + _cast, + trait, + left_paren, + right_paren, + left_square, + right_square, + greater_equal, + less_equal, + greater_than, + less_than, + not_equal, + equals, + semicolon, + dot, + comma, + plus, + _import, + minus, + multiplication, + division, + remainder, + assignment, + colon, + hat, + at, + comment, + string, + character, + integer, + word, + _while, _defer, exclamation, - arrow, - trait, - _program, - _module, - _import - ) - -var - classification: [128]TransitionClass - transitions: [16]TransitionClasses - -proc initialize_classification() -var - i: Word -begin - classification[1] := TransitionClass.eof; (* NUL *) - classification[2] := TransitionClass.invalid; (* SOH *) - classification[3] := TransitionClass.invalid; (* STX *) - classification[4] := TransitionClass.invalid; (* ETX *) - classification[5] := TransitionClass.invalid; (* EOT *) - classification[6] := TransitionClass.invalid; (* EMQ *) - classification[7] := TransitionClass.invalid; (* ACK *) - classification[8] := TransitionClass.invalid; (* BEL *) - classification[9] := TransitionClass.invalid; (* BS *) - classification[10] := TransitionClass.space; (* HT *) - classification[11] := TransitionClass.space; (* LF *) - classification[12] := TransitionClass.invalid; (* VT *) - classification[13] := TransitionClass.invalid; (* FF *) - classification[14] := TransitionClass.space; (* CR *) - classification[15] := TransitionClass.invalid; (* SO *) - classification[16] := TransitionClass.invalid; (* SI *) - classification[17] := TransitionClass.invalid; (* DLE *) - classification[18] := TransitionClass.invalid; (* DC1 *) - classification[19] := TransitionClass.invalid; (* DC2 *) - classification[20] := TransitionClass.invalid; (* DC3 *) - classification[21] := TransitionClass.invalid; (* DC4 *) - classification[22] := TransitionClass.invalid; (* NAK *) - classification[23] := TransitionClass.invalid; (* SYN *) - classification[24] := TransitionClass.invalid; (* ETB *) - classification[25] := TransitionClass.invalid; (* CAN *) - classification[26] := TransitionClass.invalid; (* EM *) - classification[27] := TransitionClass.invalid; (* SUB *) - classification[28] := TransitionClass.invalid; (* ESC *) - classification[29] := TransitionClass.invalid; (* FS *) - classification[30] := TransitionClass.invalid; (* GS *) - classification[31] := TransitionClass.invalid; (* RS *) - classification[32] := TransitionClass.invalid; (* US *) - classification[33] := TransitionClass.space; (* Space *) - classification[34] := TransitionClass.single; (* ! *) - classification[35] := TransitionClass.double_quote; (* " *) - classification[36] := TransitionClass.other; (* # *) - classification[37] := TransitionClass.other; (* $ *) - classification[38] := TransitionClass.single; (* % *) - classification[39] := TransitionClass.single; (* & *) - classification[40] := TransitionClass.single_quote; (* ' *) - classification[41] := TransitionClass.left_paren; (* ( *) - classification[42] := TransitionClass.right_paren; (* ) *) - classification[43] := TransitionClass.asterisk; (* * *) - classification[44] := TransitionClass.single; (* + *) - classification[45] := TransitionClass.single; (* , *) - classification[46] := TransitionClass.minus; (* - *) - classification[47] := TransitionClass.dot; (* . *) - classification[48] := TransitionClass.single; (* / *) - classification[49] := TransitionClass.zero; (* 0 *) - classification[50] := TransitionClass.digit; (* 1 *) - classification[51] := TransitionClass.digit; (* 2 *) - classification[52] := TransitionClass.digit; (* 3 *) - classification[53] := TransitionClass.digit; (* 4 *) - classification[54] := TransitionClass.digit; (* 5 *) - classification[55] := TransitionClass.digit; (* 6 *) - classification[56] := TransitionClass.digit; (* 7 *) - classification[57] := TransitionClass.digit; (* 8 *) - classification[58] := TransitionClass.digit; (* 9 *) - classification[59] := TransitionClass.colon; (* : *) - classification[60] := TransitionClass.single; (* ; *) - classification[61] := TransitionClass.less; (* < *) - classification[62] := TransitionClass.equals; (* = *) - classification[63] := TransitionClass.greater; (* > *) - classification[64] := TransitionClass.other; (* ? *) - classification[65] := TransitionClass.single; (* @ *) - classification[66] := TransitionClass.alpha; (* A *) - classification[67] := TransitionClass.alpha; (* B *) - classification[68] := TransitionClass.alpha; (* C *) - classification[69] := TransitionClass.alpha; (* D *) - classification[70] := TransitionClass.alpha; (* E *) - classification[71] := TransitionClass.alpha; (* F *) - classification[72] := TransitionClass.alpha; (* G *) - classification[73] := TransitionClass.alpha; (* H *) - classification[74] := TransitionClass.alpha; (* I *) - classification[75] := TransitionClass.alpha; (* J *) - classification[76] := TransitionClass.alpha; (* K *) - classification[77] := TransitionClass.alpha; (* L *) - classification[78] := TransitionClass.alpha; (* M *) - classification[79] := TransitionClass.alpha; (* N *) - classification[80] := TransitionClass.alpha; (* O *) - classification[81] := TransitionClass.alpha; (* P *) - classification[82] := TransitionClass.alpha; (* Q *) - classification[83] := TransitionClass.alpha; (* R *) - classification[84] := TransitionClass.alpha; (* S *) - classification[85] := TransitionClass.alpha; (* T *) - classification[86] := TransitionClass.alpha; (* U *) - classification[87] := TransitionClass.alpha; (* V *) - classification[88] := TransitionClass.alpha; (* W *) - classification[89] := TransitionClass.alpha; (* X *) - classification[90] := TransitionClass.alpha; (* Y *) - classification[91] := TransitionClass.alpha; (* Z *) - classification[92] := TransitionClass.single; (* [ *) - classification[93] := TransitionClass.other; (* \ *) - classification[94] := TransitionClass.single; (* ] *) - classification[95] := TransitionClass.single; (* ^ *) - classification[96] := TransitionClass.underscore; (* _ *) - classification[97] := TransitionClass.other; (* ` *) - classification[98] := TransitionClass.hex; (* a *) - classification[99] := TransitionClass.hex; (* b *) - classification[100] := TransitionClass.hex; (* c *) - classification[101] := TransitionClass.hex; (* d *) - classification[102] := TransitionClass.hex; (* e *) - classification[103] := TransitionClass.hex; (* f *) - classification[104] := TransitionClass.alpha; (* g *) - classification[105] := TransitionClass.alpha; (* h *) - classification[106] := TransitionClass.alpha; (* i *) - classification[107] := TransitionClass.alpha; (* j *) - classification[108] := TransitionClass.alpha; (* k *) - classification[109] := TransitionClass.alpha; (* l *) - classification[110] := TransitionClass.alpha; (* m *) - classification[111] := TransitionClass.alpha; (* n *) - classification[112] := TransitionClass.alpha; (* o *) - classification[113] := TransitionClass.alpha; (* p *) - classification[114] := TransitionClass.alpha; (* q *) - classification[115] := TransitionClass.alpha; (* r *) - classification[116] := TransitionClass.alpha; (* s *) - classification[117] := TransitionClass.alpha; (* t *) - classification[118] := TransitionClass.alpha; (* u *) - classification[119] := TransitionClass.alpha; (* v *) - classification[120] := TransitionClass.alpha; (* w *) - classification[121] := TransitionClass.x; (* x *) - classification[122] := TransitionClass.alpha; (* y *) - classification[123] := TransitionClass.alpha; (* z *) - classification[124] := TransitionClass.other; (* { *) - classification[125] := TransitionClass.single; (* | *) - classification[126] := TransitionClass.other; (* } *) - classification[127] := TransitionClass.single; (* ~ *) - classification[128] := TransitionClass.invalid; (* DEL *) - - i := 129u; - while i <= 256u do - classification[i] := TransitionClass.other; - i := i + 1u - end -end - -proc compare_keyword(keyword: String; token_start: BufferPosition; token_end: ^Char): Bool -var - result: Bool - index: Word - continue: Bool -begin - index := 0u; - result := true; - continue := (index < keyword.length) & (token_start.iterator <> token_end); - - while continue & result do - result := keyword[index] = token_start.iterator^ - or cast(tolower(cast(keyword[index]: Int)): Char) = token_start.iterator^; - token_start.iterator := token_start.iterator + 1; - index := index + 1u; - continue := (index < keyword.length) & (token_start.iterator <> token_end) - end; - result := result & index = keyword.length; - - return result & (token_start.iterator = token_end) -end - -(* Reached the end of file. *) -proc transition_action_eof(lexer: ^Lexer; token: ^LexerToken) -begin - token^.kind := LexerKind.unknown -end - -proc increment(position: ^BufferPosition) -begin - position^.iterator := position^.iterator + 1 -end - -(* Add the character to the token currently read and advance to the next character. *) -proc transition_action_accumulate(lexer: ^Lexer; token: ^LexerToken) -begin - increment(@lexer^.current) -end - -(* The current character is not a part of the token. Finish the token already - * read. Don't advance to the next character. *) -proc transition_action_finalize(lexer: ^Lexer; token: ^LexerToken) -begin - if lexer^.start.iterator^ = ':' then - token^.kind := LexerKind.colon - end; - if lexer^.start.iterator^ = '>' then - token^.kind := LexerKind.greater_than - end; - if lexer^.start.iterator^ = '<' then - token^.kind := LexerKind.less_than - end; - if lexer^.start.iterator^ = '(' then - token^.kind := LexerKind.left_paren - end; - if lexer^.start.iterator^ = '-' then - token^.kind := LexerKind.minus - end; - if lexer^.start.iterator^ = '.' then - token^.kind := LexerKind.dot - end -end - -(* An action for tokens containing multiple characters. *) -proc transition_action_composite(lexer: ^Lexer; token: ^LexerToken) -begin - if lexer^.start.iterator^ = '<' then - if lexer^.current.iterator^ = '>' then - token^.kind := LexerKind.not_equal - end; - if lexer^.current.iterator^ = '=' then - token^.kind := LexerKind.less_equal - end - end; - if (lexer^.start.iterator^ = '>') & (lexer^.current.iterator^ = '=') then - token^.kind := LexerKind.greater_equal - end; - if (lexer^.start.iterator^ = ':') & (lexer^.current.iterator^ = '=') then - token^.kind := LexerKind.assignment - end; - if (lexer^.start.iterator^ = '-') & (lexer^.current.iterator^ = '>') then - token^.kind := LexerKind.arrow - end; - increment(@lexer^.current) -end - -(* Skip a space. *) -proc transition_action_skip(lexer: ^Lexer; token: ^LexerToken) -begin - increment(@lexer^.start); - - if lexer^.start.iterator^ = '\n' then - lexer^.start.location.line := lexer^.start.location.line + 1u; - lexer^.start.location.column := 1u - end; - lexer^.current := lexer^.start -end - -(* Delimited string action. *) -proc transition_action_delimited(lexer: ^Lexer; token: ^LexerToken) -var - text_length: Word -begin - if lexer^.start.iterator^ = '(' then - token^.kind := LexerKind.comment - end; - if lexer^.start.iterator^ = '"' then - text_length := cast(lexer^.current.iterator - lexer^.start.iterator + 1: Word); - - token^.value.stringKind := String(cast(malloc(text_length): ^Char), text_length); - memcpy(token^.value.stringKind.ptr, lexer^.start.iterator, text_length); - - token^.kind := LexerKind.character - end; - if lexer^.start.iterator^ = '\'' then - text_length := cast(lexer^.current.iterator - lexer^.start.iterator + 1: Word); - - token^.value.stringKind := String(cast(malloc(text_length): ^Char), text_length); - memcpy(token^.value.stringKind.ptr, lexer^.start.iterator, text_length); - - token^.kind := LexerKind.string - end; - increment(@lexer^.current) -end - -(* Finalize keyword or identifier. *) -proc transition_action_key_id(lexer: ^Lexer; token: ^LexerToken) -begin - token^.kind := LexerKind.identifier; - - token^.value.identifierKind[1] := cast(lexer^.current.iterator - lexer^.start.iterator: Char); - memcpy(@token^.value.identifierKind[2], lexer^.start.iterator, cast(token^.value.identifierKind[1]: Word)); - - if compare_keyword("program", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._program - end; - if compare_keyword("import", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._import - end; - if compare_keyword("const", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._const - end; - if compare_keyword("var", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._var - end; - if compare_keyword("if", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._if - end; - if compare_keyword("then", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._then - end; - if compare_keyword("elsif", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._elsif - end; - if compare_keyword("else", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._else - end; - if compare_keyword("while", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._while - end; - if compare_keyword("do", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._do - end; - if compare_keyword("proc", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._proc - end; - if compare_keyword("begin", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._begin - end; - if compare_keyword("end", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._end - end; - if compare_keyword("type", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._type - end; - if compare_keyword("record", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._record - end; - if compare_keyword("union", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._union - end; - if compare_keyword("NIL", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind.null - end; - if compare_keyword("or", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._or - end; - if compare_keyword("return", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._return - end; - if compare_keyword("defer", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._defer - end; - if compare_keyword("TO", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind.to - end; - if compare_keyword("CASE", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._case - end; - if compare_keyword("OF", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._of - end; - if compare_keyword("module", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._module - end; - if compare_keyword("xor", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind._xor - end; - if compare_keyword("TRUE", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind.boolean; - token^.value.booleanKind := true - end; - if compare_keyword("FALSE", lexer^.start, lexer^.current.iterator) then - token^.kind := LexerKind.boolean; - token^.value.booleanKind := false - end -end - -(* Action for tokens containing only one character. The character cannot be - * followed by other characters forming a composite token. *) -proc transition_action_single(lexer: ^Lexer; token: ^LexerToken) -begin - if lexer^.current.iterator^ = '&' then - token^.kind := LexerKind.and - end; - if lexer^.current.iterator^ = ';' then - token^.kind := LexerKind.semicolon - end; - if lexer^.current.iterator^ = ',' then - token^.kind := LexerKind.comma - end; - if lexer^.current.iterator^ = '~' then - token^.kind := LexerKind.not - end; - if lexer^.current.iterator^ = ')' then - token^.kind := LexerKind.right_paren - end; - if lexer^.current.iterator^ = '[' then - token^.kind := LexerKind.left_square - end; - if lexer^.current.iterator^ = ']' then - token^.kind := LexerKind.right_square - end; - if lexer^.current.iterator^ = '^' then - token^.kind := LexerKind.hat - end; - if lexer^.current.iterator^ = '=' then - token^.kind := LexerKind.equal - end; - if lexer^.current.iterator^ = '+' then - token^.kind := LexerKind.plus - end; - if lexer^.current.iterator^ = '*' then - token^.kind := LexerKind.multiplication - end; - if lexer^.current.iterator^ = '/' then - token^.kind := LexerKind.division - end; - if lexer^.current.iterator^ = '%' then - token^.kind := LexerKind.remainder - end; - if lexer^.current.iterator^ = '@' then - token^.kind := LexerKind.at - end; - if lexer^.current.iterator^ = '|' then - token^.kind := LexerKind.pipe - end; - increment(@lexer^.current) -end - -(* Handle an integer literal. *) -proc transition_action_integer(lexer: ^Lexer; token: ^LexerToken) -var - buffer: String - integer_length: Word - found: Bool -begin - token^.kind := LexerKind.integer; - - integer_length := cast(lexer^.current.iterator - lexer^.start.iterator: Word); - memset(cast(token^.value.identifierKind.ptr: Pointer), 0, #size(Identifier)); - memcpy(@token^.value.identifierKind[1], lexer^.start.iterator, integer_length); - - token^.value.identifierKind[cast(token^.value.identifierKind[1]: Int) + 2] := '\0'; - token^.value.integerKind := atoi(@token^.value.identifierKind[2]) -end - -proc set_default_transition(current_state: TransitionState; default_action: TransitionAction; next_state: TransitionState): Int -var - default_transition: Transition - state_index: Int -begin - default_transition.action := default_action; - default_transition.next_state := next_state; - state_index := cast(current_state: Int) + 1; - - transitions[state_index][cast(TransitionClass.invalid: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.digit: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.alpha: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.space: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.colon: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.equals: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.left_paren: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.right_paren: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.underscore: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.single: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.hex: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.zero: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.x: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.eof: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.dot: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.minus: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.single_quote: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.double_quote: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.greater: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.less: Int) + 1] := default_transition; - transitions[state_index][cast(TransitionClass.other: Int) + 1] := default_transition; - - return state_index -end - -(* - * The transition table describes transitions from one state to another, given - * a symbol (character class). - * - * The table has m rows and n columns, where m is the amount of states and n is - * the amount of classes. So given the current state and a classified character - * the table can be used to look up the next state. - * - * Each cell is a word long. - * - The least significant byte of the word is a row number (beginning with 0). - * It specifies the target state. "ff" means that this is an end state and no - * transition is possible. - * - The next byte is the action that should be performed when transitioning. - * For the meaning of actions see labels in the lex_next function, which - * handles each action. - *) -proc initialize_transitions() -var - state_index: Int -begin - (* Start state. *) - state_index := cast(TransitionState.start: Int) + 1; - - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.decimal; - - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.space: Int) + 1].action := transition_action_skip; - transitions[state_index][cast(TransitionClass.space: Int) + 1].next_state := TransitionState.start; - - transitions[state_index][cast(TransitionClass.colon: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.colon: Int) + 1].next_state := TransitionState.colon; - - transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_single; - transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.left_paren: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.left_paren: Int) + 1].next_state := TransitionState.left_paren; - - transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].action := transition_action_single; - transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_single; - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.single: Int) + 1].action := transition_action_single; - transitions[state_index][cast(TransitionClass.single: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.leading_zero; - - transitions[state_index][cast(TransitionClass.x: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := transition_action_eof; - transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.dot: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.dot: Int) + 1].next_state := TransitionState.dot; - - transitions[state_index][cast(TransitionClass.minus: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.minus: Int) + 1].next_state := TransitionState.minus; - - transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].next_state := TransitionState.character; - - transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].next_state := TransitionState.string; - - transitions[state_index][cast(TransitionClass.greater: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.greater: Int) + 1].next_state := TransitionState.greater; - - transitions[state_index][cast(TransitionClass.less: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.less: Int) + 1].next_state := TransitionState.less; - - transitions[state_index][cast(TransitionClass.other: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.other: Int) + 1].next_state := TransitionState.finish; - - (* Colon state. *) - state_index := set_default_transition(TransitionState.colon, transition_action_finalize, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_composite; - transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish; - - (* Identifier state. *) - state_index := set_default_transition(TransitionState.identifier, transition_action_key_id, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.identifier; - - transitions[state_index][cast(TransitionClass.x: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.identifier; - - (* Decimal state. *) - state_index := set_default_transition(TransitionState.decimal, transition_action_integer, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.decimal; - - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.decimal_suffix; - - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.decimal_suffix; - - transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.decimal; - - transitions[state_index][cast(TransitionClass.x: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.decimal_suffix; - - (* Greater state. *) - state_index := set_default_transition(TransitionState.greater, transition_action_finalize, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_composite; - transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish; - - (* Minus state. *) - state_index := set_default_transition(TransitionState.minus, transition_action_finalize, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.greater: Int) + 1].action := transition_action_composite; - transitions[state_index][cast(TransitionClass.greater: Int) + 1].next_state := TransitionState.finish; - - (* Left paren state. *) - state_index := set_default_transition(TransitionState.left_paren, transition_action_finalize, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.comment; - - (* Less state. *) - state_index := set_default_transition(TransitionState.less, transition_action_finalize, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.equals: Int) + 1].action := transition_action_composite; - transitions[state_index][cast(TransitionClass.equals: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.greater: Int) + 1].action := transition_action_composite; - transitions[state_index][cast(TransitionClass.greater: Int) + 1].next_state := TransitionState.finish; - - (* Hexadecimal after 0x. *) - state_index := set_default_transition(TransitionState.dot, transition_action_finalize, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.dot: Int) + 1].action := transition_action_composite; - transitions[state_index][cast(TransitionClass.dot: Int) + 1].next_state := TransitionState.finish; - - (* Comment. *) - state_index := set_default_transition(TransitionState.comment, transition_action_accumulate, TransitionState.comment); - - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.closing_comment; - - transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish; - - (* Closing comment. *) - state_index := set_default_transition(TransitionState.closing_comment, transition_action_accumulate, TransitionState.comment); - - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].action := transition_action_delimited; - transitions[state_index][cast(TransitionClass.right_paren: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].action := transition_action_accumulate; - transitions[state_index][cast(TransitionClass.asterisk: Int) + 1].next_state := TransitionState.closing_comment; - - transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish; - - (* Character. *) - state_index := set_default_transition(TransitionState.character, transition_action_accumulate, TransitionState.character); - - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].action := transition_action_delimited; - transitions[state_index][cast(TransitionClass.single_quote: Int) + 1].next_state := TransitionState.finish; - - (* String. *) - state_index := set_default_transition(TransitionState.string, transition_action_accumulate, TransitionState.string); - - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.invalid: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.eof: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.eof: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].action := transition_action_delimited; - transitions[state_index][cast(TransitionClass.double_quote: Int) + 1].next_state := TransitionState.finish; - - (* Leading zero. *) - state_index := set_default_transition(TransitionState.leading_zero, transition_action_integer, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.underscore: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.x: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.finish; - - (* Digit with a character suffix. *) - state_index := set_default_transition(TransitionState.decimal_suffix, transition_action_integer, TransitionState.finish); - - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.alpha: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.digit: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.digit: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.hex: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.hex: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.zero: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.zero: Int) + 1].next_state := TransitionState.finish; - - transitions[state_index][cast(TransitionClass.x: Int) + 1].action := nil; - transitions[state_index][cast(TransitionClass.x: Int) + 1].next_state := TransitionState.finish -end - -proc lexer_make*(lexer: ^Lexer; input: ^FILE) -begin - lexer^.input := input; - lexer^.length := 0u; - - lexer^.buffer := cast(malloc(CHUNK_SIZE): ^Char); - memset(cast(lexer^.buffer: Pointer), 0, CHUNK_SIZE); - lexer^.size := CHUNK_SIZE -end - -(* Returns the last read token. *) -proc lexer_current*(lexer: ^Lexer): LexerToken -var - current_class: TransitionClass - current_state: TransitionState - current_transition: Transition - result: LexerToken - index1: Word - index2: Word -begin - lexer^.current := lexer^.start; - current_state := TransitionState.start; - - while current_state <> TransitionState.finish do - index1 := cast(lexer^.current.iterator^: Word) + 1u; - current_class := classification[index1]; - - index1 := cast(current_state: Word) + 1u; - index2 := cast(current_class: Word) + 1u; - - current_transition := transitions[index1][index2]; - if current_transition.action <> nil then - current_transition.action(lexer, @result) - end; - current_state := current_transition.next_state - end; - result.start_location := lexer^.start.location; - result.end_location := lexer^.current.location; - - return result -end - -(* Read and return the next token. *) -proc lexer_lex*(lexer: ^Lexer): LexerToken -var - result: LexerToken -begin - if lexer^.length = 0u then - lexer^.length := fread(cast(lexer^.buffer: Pointer), CHUNK_SIZE, 1u, lexer^.input); - lexer^.current.location.column := 1u; - lexer^.current.location.line := 1u; - lexer^.current.iterator := lexer^.buffer - end; - lexer^.start := lexer^.current; - - result := lexer_current(lexer); - return result -end - -proc lexer_destroy*(lexer: ^Lexer) -begin - free(cast(lexer^.buffer: Pointer)) -end - -proc lexer_initialize() -begin - initialize_classification(); - initialize_transitions() -end + shift_right, + shift_left, + pipe, + _case, + _do, + _of, + eof + ) + (** + * Classification table assigns each possible character to a group (class). All + * characters of the same group a handled equivalently. + *) + ElnaLexerClass = ( + invalid, + digit, + alpha, + space, + colon, + equals, + left_paren, + right_paren, + asterisk, + backslash, + single, + hex, + zero, + x, + eof, + dot, + minus, + single_quote, + double_quote, + greater, + less, + other, + number_sign + ) end. diff --git a/source/main.elna b/source/main.elna index aeaba2f..781d681 100644 --- a/source/main.elna +++ b/source/main.elna @@ -24,18 +24,9 @@ type advance: proc(stream: Pointer); head: proc(stream: Pointer): Char end - Token* = record - kind: LexerKind; - value: union - int_value: Int; - string: String; - boolean_value: Bool; - char_value: Char - end - end Tokenizer* = record length: Word; - data: ^Token + data: ^^ElnaLexerToken end (* @@ -57,7 +48,7 @@ proc string_dup(origin: String): String var copy: ^Char begin - copy := cast(malloc(origin.length): ^Char); + copy := malloc(origin.length); strncpy(copy, origin.ptr, origin.length); return String(copy, origin.length) @@ -326,92 +317,109 @@ begin end (* Categorize an identifier. *) -proc lexer_categorize(token_content: String): Token +proc lexer_categorize(token_content: String): ^ElnaLexerToken var - current_token: Token + current_token: ^ElnaLexerToken begin if token_content = "if" then - current_token.kind := LexerKind._if + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._if elsif token_content = "then" then - current_token.kind := LexerKind._then + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._then elsif token_content = "else" then - current_token.kind := LexerKind._else + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._else elsif token_content = "elsif" then - current_token.kind := LexerKind._elsif + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._elsif elsif token_content = "while" then - current_token.kind := LexerKind._while + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._while elsif token_content = "do" then - current_token.kind := LexerKind._do + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._do elsif token_content = "proc" then - current_token.kind := LexerKind._proc + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._proc elsif token_content = "begin" then - current_token.kind := LexerKind._begin + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._begin elsif token_content = "end" then - current_token.kind := LexerKind._end + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._end elsif token_content = "extern" then - current_token.kind := LexerKind._extern + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._extern elsif token_content = "const" then - current_token.kind := LexerKind._const + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._const elsif token_content = "var" then - current_token.kind := LexerKind._var + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._var elsif token_content = "case" then - current_token.kind := LexerKind._case + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._case elsif token_content = "of" then - current_token.kind := LexerKind._of + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._of elsif token_content = "type" then - current_token.kind := LexerKind._type + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._type elsif token_content = "record" then - current_token.kind := LexerKind._record - elsif token_content = "union" then - current_token.kind := LexerKind._union + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._record elsif token_content = "true" then - current_token.kind := LexerKind.boolean; - current_token.value.boolean_value := true + current_token := malloc(#size(ElnaLexerBooleanToken)); + current_token^.kind := ElnaLexerKind.boolean; + cast(current_token: ^ElnaLexerBooleanToken)^.value := true elsif token_content = "false" then - current_token.kind := LexerKind.boolean; - current_token.value.boolean_value := false + current_token := malloc(#size(ElnaLexerBooleanToken)); + current_token^.kind := ElnaLexerKind.boolean; + cast(current_token: ^ElnaLexerBooleanToken)^.value := false elsif token_content = "nil" then - current_token.kind := LexerKind.null + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.null elsif token_content = "or" then - current_token.kind := LexerKind._or + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._or elsif token_content = "return" then - current_token.kind := LexerKind._return + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._return elsif token_content = "cast" then - current_token.kind := LexerKind._cast + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._cast elsif token_content = "defer" then - current_token.kind := LexerKind._defer - elsif token_content = "program" then - current_token.kind := LexerKind._program - elsif token_content = "module" then - current_token.kind := LexerKind._module + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._defer elsif token_content = "import" then - current_token.kind := LexerKind._import + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind._import else - current_token.kind := LexerKind.identifier; - current_token.value.string := string_dup(token_content) + current_token := malloc(#size(ElnaLexerStringToken)); + current_token^.kind := ElnaLexerKind.identifier; + cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(token_content) end; return current_token end -proc lexer_add_token(lexer: ^Tokenizer; token: Token) +proc lexer_add_token(lexer: ^Tokenizer; token: ^ElnaLexerToken) var new_length: Word begin new_length := lexer^.length + 1u; - lexer^.data := cast(reallocarray(cast(lexer^.data: Pointer), new_length, #size(Token)): ^Token); + lexer^.data := cast(reallocarray(cast(lexer^.data: Pointer), new_length, #size(Pointer)): ^^ElnaLexerToken); (lexer^.data + lexer^.length)^ := token; lexer^.length := new_length end (* Read the next token from the input. *) -proc lexer_next(source_code: SourceCode; token_buffer: ^StringBuffer): Token +proc lexer_next(source_code: SourceCode; token_buffer: ^StringBuffer): ^ElnaLexerToken var - current_token: Token + current_token: ^ElnaLexerToken := nil first_char: Char begin - current_token.kind := LexerKind.unknown; - first_char := source_code_head(source_code); if isalpha(cast(first_char: Int)) <> 0 or first_char = '_' then @@ -421,158 +429,170 @@ begin source_code_advance(@source_code); lexer_identifier(@source_code, token_buffer); - current_token.kind := LexerKind.trait; - current_token.value.string := string_dup(string_buffer_clear(token_buffer)) + current_token := malloc(#size(ElnaLexerStringToken)); + current_token^.kind := ElnaLexerKind.trait; + cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(string_buffer_clear(token_buffer)) elsif isdigit(cast(first_char: Int)) <> 0 then - lexer_number(@source_code, @current_token.value.int_value); + current_token := malloc(#size(ElnaLexerIntegerToken)); + lexer_number(@source_code, @cast(current_token: ^ElnaLexerIntegerToken)^.value); if source_code_expect(@source_code, 'u') then - current_token.kind := LexerKind.word; - source_code_advance(@source_code) + current_token^.kind := ElnaLexerKind.word; + source_code_advance(@source_code) else - current_token.kind := LexerKind.integer + current_token^.kind := ElnaLexerKind.integer end elsif first_char = '(' then source_code_advance(@source_code); if source_code_empty(@source_code) then - current_token.kind := LexerKind.left_paren + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.left_paren elsif source_code_head(source_code) = '*' then source_code_advance(@source_code); if lexer_comment(@source_code, token_buffer) then - current_token.value.string := string_dup(string_buffer_clear(token_buffer)); - current_token.kind := LexerKind.comment - else - current_token.kind := LexerKind.unknown + current_token := malloc(#size(ElnaLexerStringToken)); + cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(string_buffer_clear(token_buffer)); + current_token^.kind := ElnaLexerKind.comment end else - current_token.kind := LexerKind.left_paren + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.left_paren end elsif first_char = ')' then - current_token.kind := LexerKind.right_paren; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.right_paren; source_code_advance(@source_code) elsif first_char = '\'' then source_code_advance(@source_code); - if lexer_character(@source_code, @current_token.value.char_value) & source_code_expect(@source_code, '\'') then - current_token.kind := LexerKind.character; + if lexer_character(@source_code, @first_char) & source_code_expect(@source_code, '\'') then + current_token := malloc(#size(ElnaLexerCharacterToken)); + current_token^.kind := ElnaLexerKind.character; + cast(current_token: ^ElnaLexerCharacterToken)^.value := first_char; source_code_advance(@source_code) - else - current_token.kind := LexerKind.unknown end elsif first_char = '"' then source_code_advance(@source_code); if lexer_string(@source_code, token_buffer) then - current_token.kind := LexerKind.string; - current_token.value.string := string_dup(string_buffer_clear(token_buffer)) - else - current_token.kind := LexerKind.unknown + current_token := malloc(#size(ElnaLexerStringToken)); + current_token^.kind := ElnaLexerKind.string; + cast(current_token: ^ElnaLexerStringToken)^.value := string_dup(string_buffer_clear(token_buffer)) end elsif first_char = '[' then - current_token.kind := LexerKind.left_square; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.left_square; source_code_advance(@source_code) elsif first_char = ']' then - current_token.kind := LexerKind.right_square; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.right_square; source_code_advance(@source_code) elsif first_char = '>' then source_code_advance(@source_code); + current_token := malloc(#size(ElnaLexerToken)); if source_code_empty(@source_code) then - current_token.kind := LexerKind.greater_than + current_token^.kind := ElnaLexerKind.greater_than elsif source_code_head(source_code) = '=' then - current_token.kind := LexerKind.greater_equal; + current_token^.kind := ElnaLexerKind.greater_equal; source_code_advance(@source_code) elsif source_code_head(source_code) = '>' then - current_token.kind := LexerKind.shift_right; + current_token^.kind := ElnaLexerKind.shift_right; source_code_advance(@source_code) else - current_token.kind := LexerKind.greater_than + current_token^.kind := ElnaLexerKind.greater_than end elsif first_char = '<' then source_code_advance(@source_code); + current_token := malloc(#size(ElnaLexerToken)); if source_code_empty(@source_code) then - current_token.kind := LexerKind.less_than + current_token^.kind := ElnaLexerKind.less_than elsif source_code_head(source_code) = '=' then - current_token.kind := LexerKind.less_equal; + current_token^.kind := ElnaLexerKind.less_equal; source_code_advance(@source_code) elsif source_code_head(source_code) = '<' then - current_token.kind := LexerKind.shift_left; + current_token^.kind := ElnaLexerKind.shift_left; source_code_advance(@source_code) elsif source_code_head(source_code) = '>' then - current_token.kind := LexerKind.not_equal; + current_token^.kind := ElnaLexerKind.not_equal; source_code_advance(@source_code) else - current_token.kind := LexerKind.less_than + current_token^.kind := ElnaLexerKind.less_than end elsif first_char = '=' then - current_token.kind := LexerKind.equal; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.equals; source_code_advance(@source_code) elsif first_char = ';' then - current_token.kind := LexerKind.semicolon; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.semicolon; source_code_advance(@source_code) elsif first_char = '.' then - current_token.kind := LexerKind.dot; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.dot; source_code_advance(@source_code) elsif first_char = ',' then - current_token.kind := LexerKind.comma; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.comma; source_code_advance(@source_code) elsif first_char = '+' then - current_token.kind := LexerKind.plus; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.plus; source_code_advance(@source_code) elsif first_char = '-' then source_code_advance(@source_code); - - if source_code_empty(@source_code) then - current_token.kind := LexerKind.minus - elsif source_code_head(source_code) = '>' then - current_token.kind := LexerKind.arrow; - source_code_advance(@source_code) - else - current_token.kind := LexerKind.minus - end + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.minus elsif first_char = '*' then - current_token.kind := LexerKind.multiplication; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.multiplication; source_code_advance(@source_code) elsif first_char = '/' then - current_token.kind := LexerKind.division; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.division; source_code_advance(@source_code) elsif first_char = '%' then - current_token.kind := LexerKind.remainder; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.remainder; source_code_advance(@source_code) elsif first_char = ':' then source_code_advance(@source_code); + current_token := malloc(#size(ElnaLexerToken)); if source_code_empty(@source_code) then - current_token.kind := LexerKind.colon + current_token^.kind := ElnaLexerKind.colon elsif source_code_head(source_code) = '=' then - current_token.kind := LexerKind.assignment; + current_token^.kind := ElnaLexerKind.assignment; source_code_advance(@source_code) else - current_token.kind := LexerKind.colon + current_token^.kind := ElnaLexerKind.colon end elsif first_char = '^' then - current_token.kind := LexerKind.hat; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.hat; source_code_advance(@source_code) elsif first_char = '@' then - current_token.kind := LexerKind.at; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.at; source_code_advance(@source_code) elsif first_char = '!' then - current_token.kind := LexerKind.exclamation; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.exclamation; source_code_advance(@source_code) elsif first_char = '&' then - current_token.kind := LexerKind.and; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.and; source_code_advance(@source_code) elsif first_char = '~' then - current_token.kind := LexerKind.not; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.not; source_code_advance(@source_code) elsif first_char = '|' then - current_token.kind := LexerKind.pipe; - source_code_advance(@source_code) - else - current_token.kind := LexerKind.unknown; + current_token := malloc(#size(ElnaLexerToken)); + current_token^.kind := ElnaLexerKind.pipe; source_code_advance(@source_code) end; @@ -582,7 +602,7 @@ end (* Split the source text into tokens. *) proc lexer_text(source_code: SourceCode): Tokenizer var - current_token: Token + current_token: ^ElnaLexerToken token_buffer: StringBuffer lexer: Tokenizer begin @@ -594,7 +614,7 @@ begin while ~source_code_empty(@source_code) do current_token := lexer_next(source_code, @token_buffer); - if current_token.kind <> LexerKind.unknown then + if current_token <> nil then lexer_add_token(@lexer, current_token); lexer_spaces(@source_code) else @@ -611,151 +631,141 @@ end Parser. *) -proc parse(tokens: ^Token; tokens_size: Word) +proc parse(tokens: ^^ElnaLexerToken; tokens_size: Word) var - current_token: ^Token + current_token: ^ElnaLexerToken i: Word := 0u begin while i < tokens_size do - current_token := tokens + i; + current_token := (tokens + i)^; case current_token^.kind of - LexerKind._if: + ElnaLexerKind._if: write_s("IF") - | LexerKind._then: + | ElnaLexerKind._then: write_s("THEN") - | LexerKind._else: + | ElnaLexerKind._else: write_s("ELSE") - | LexerKind._elsif: + | ElnaLexerKind._elsif: write_s("ELSIF") - | LexerKind._while: + | ElnaLexerKind._while: write_s("WHILE") - | LexerKind._do: + | ElnaLexerKind._do: write_s("DO") - | LexerKind._proc: + | ElnaLexerKind._proc: write_s("PROC") - | LexerKind._begin: + | ElnaLexerKind._begin: write_s("BEGIN") - | LexerKind._end: + | ElnaLexerKind._end: write_s("END") - | LexerKind._extern: + | ElnaLexerKind._extern: write_s("EXTERN") - | LexerKind._const: + | ElnaLexerKind._const: write_s("CONST") - | LexerKind._var: + | ElnaLexerKind._var: write_s("VAR") - | LexerKind._case: + | ElnaLexerKind._case: write_s("CASE") - | LexerKind._of: + | ElnaLexerKind._of: write_s("OF") - | LexerKind._type: + | ElnaLexerKind._type: write_s("TYPE") - | LexerKind._record: + | ElnaLexerKind._record: write_s("RECORD") - | LexerKind._union: - write_s("UNION") - | LexerKind.pipe: + | ElnaLexerKind.pipe: write_s("|") - | LexerKind.to: - write_s("TO") - | LexerKind.boolean: + | ElnaLexerKind.boolean: write_s("BOOLEAN<"); - write_b(current_token^.value.boolean_value); + write_b(cast(current_token: ^ElnaLexerBooleanToken)^.value); write_c('>') - | LexerKind.null: + | ElnaLexerKind.null: write_s("NIL") - | LexerKind.and: + | ElnaLexerKind.and: write_s("&") - | LexerKind._or: + | ElnaLexerKind._or: write_s("OR") - | LexerKind.not: + | ElnaLexerKind.not: write_s("~") - | LexerKind._return: + | ElnaLexerKind._return: write_s("RETURN") - | LexerKind._cast: + | ElnaLexerKind._cast: write_s("CAST") - | LexerKind.shift_left: + | ElnaLexerKind.shift_left: write_s("<<") - | LexerKind.shift_right: + | ElnaLexerKind.shift_right: write_s(">>") - | LexerKind.identifier: + | ElnaLexerKind.identifier: write_c('<'); - write_s(current_token^.value.string); + write_s(cast(current_token: ^ElnaLexerStringToken)^.value); write_c('>') - | LexerKind.trait: + | ElnaLexerKind.trait: write_c('#'); - write_s(current_token^.value.string) - | LexerKind.left_paren: + write_s(cast(current_token: ^ElnaLexerStringToken)^.value) + | ElnaLexerKind.left_paren: write_s("(") - | LexerKind.right_paren: + | ElnaLexerKind.right_paren: write_s(")") - | LexerKind.left_square: + | ElnaLexerKind.left_square: write_s("[") - | LexerKind.right_square: + | ElnaLexerKind.right_square: write_s("]") - | LexerKind.greater_equal: + | ElnaLexerKind.greater_equal: write_s(">=") - | LexerKind.less_equal: + | ElnaLexerKind.less_equal: write_s("<=") - | LexerKind.greater_than: + | ElnaLexerKind.greater_than: write_s(">") - | LexerKind.less_than: + | ElnaLexerKind.less_than: write_s("<") - | LexerKind.equal: + | ElnaLexerKind.equals: write_s("=") - | LexerKind.not_equal: + | ElnaLexerKind.not_equal: write_s("<>") - | LexerKind.semicolon: + | ElnaLexerKind.semicolon: write_c(';') - | LexerKind.dot: + | ElnaLexerKind.dot: write_c('.') - | LexerKind.comma: + | ElnaLexerKind.comma: write_c(',') - | LexerKind.plus: + | ElnaLexerKind.plus: write_c('+') - | LexerKind.minus: + | ElnaLexerKind.minus: write_c('-') - | LexerKind.multiplication: + | ElnaLexerKind.multiplication: write_c('*') - | LexerKind.division: + | ElnaLexerKind.division: write_c('/') - | LexerKind.remainder: + | ElnaLexerKind.remainder: write_c('%') - | LexerKind.assignment: + | ElnaLexerKind.assignment: write_s(":=") - | LexerKind.colon: + | ElnaLexerKind.colon: write_c(':') - | LexerKind.hat: + | ElnaLexerKind.hat: write_c('^') - | LexerKind.at: + | ElnaLexerKind.at: write_c('@') - | LexerKind.comment: + | ElnaLexerKind.comment: write_s("(* COMMENT *)") - | LexerKind.integer: + | ElnaLexerKind.integer: write_c('<'); - write_i(current_token^.value.int_value); + write_i(cast(current_token: ^ElnaLexerIntegerToken)^.value); write_c('>') - | LexerKind.word: + | ElnaLexerKind.word: write_c('<'); - write_i(current_token^.value.int_value); + write_i(cast(current_token: ^ElnaLexerIntegerToken)^.value); write_s("u>") - | LexerKind.character: + | ElnaLexerKind.character: write_c('<'); - write_i(cast(current_token^.value.char_value: Int)); + write_i(cast(cast(current_token: ^ElnaLexerCharacterToken)^.value: Int)); write_s("c>") - | LexerKind.string: + | ElnaLexerKind.string: write_s("\"...\"") - | LexerKind._defer: + | ElnaLexerKind._defer: write_s("DEFER") - | LexerKind.exclamation: + | ElnaLexerKind.exclamation: write_c('!') - | LexerKind.arrow: - write_s("->") - | LexerKind._program: - write_s("PROGRAM") - | LexerKind._module: - write_s("MODULE") - | LexerKind._import: + | ElnaLexerKind._import: write_s("IMPORT") else write_s("UNKNOWN<"); @@ -790,7 +800,7 @@ end proc process(argc: Int; argv: ^^Char): Int var - tokens: ^Token + tokens: ^ElnaLexerToken tokens_size: Word source_code: SourceCode command_line: ^CommandLine @@ -817,7 +827,7 @@ begin end; source_code.position := TextLocation(1u, 1u); - source_code.input := cast(source_file: Pointer); + source_code.input := source_file; source_code.empty := source_file_empty; source_code.head := source_file_head; source_code.advance := source_file_advance; |
