diff options
Diffstat (limited to 'source/lexer.elna')
| -rw-r--r-- | source/lexer.elna | 815 |
1 files changed, 0 insertions, 815 deletions
diff --git a/source/lexer.elna b/source/lexer.elna deleted file mode 100644 index 71e76a7..0000000 --- a/source/lexer.elna +++ /dev/null @@ -1,815 +0,0 @@ -(* This Source Code Form is subject to the terms of the Mozilla Public License, - v. 2.0. If a copy of the MPL was not distributed with this file, You can - obtain one at https://mozilla.org/MPL/2.0/. *) -module -import sys.posix.cstdio, common, sys.posix.cstdlib - -type - ElnaLexerState* = ( - start, - colon, - identifier, - decimal, - leading_zero, - greater, - minus, - left_paren, - less, - dot, - comment, - closing_comment, - character, - character_escape, - string, - string_escape, - number_sign, - trait, - finish - ) - ElnaLexerToken* = record - kind: ElnaLexerKind; - start: []const Word8; - position: ElnaPosition - end - ElnaLexerBooleanToken* = record(ElnaLexerToken) - value: Bool - end - ElnaLexerCharacterToken* = record(ElnaLexerToken) - value: Word8 - end - ElnaLexerStringToken* = record(ElnaLexerToken) - value: []const Word8 - end - ElnaLexerIntegerToken* = record(ElnaLexerToken) - value: Int - end - - TransitionAction = proc(lexer: ^Lexer; token: ^ElnaLexerToken) - ElnaLexerAction = (none, accumulate, skip, single, eof, finalize, composite, key_id, integer, delimited) - - ElnaLexerTransition = record - action: ElnaLexerAction; - next_state: ElnaLexerState - end - - ElnaLexerCursor* = record - state: ElnaLexerState; - start: ^Word8; - finish: ^Word8; - token: ^ElnaLexerToken; - position: ElnaPosition - end - - BufferPosition* = record - iterator: ^Word8; - location: ElnaLocation - end - Lexer* = record - input: ^FILE; - buffer: ^Word8; - size: Word; - length: Word; - start: BufferPosition; - current: BufferPosition - end - ElnaLexerKind* = ( - identifier, - _const, - _var, - _proc, - _type, - _begin, - _end, - _if, - _then, - _else, - _elsif, - _extern, - _record, - boolean, - null, - and, - _or, - _xor, - not, - _return, - _cast, - trait, - left_paren, - right_paren, - left_square, - right_square, - left_brace, - right_brace, - greater_equal, - less_equal, - greater_than, - less_than, - not_equal, - equals, - semicolon, - dot, - comma, - plus, - _import, - minus, - multiplication, - division, - remainder, - assignment, - colon, - hat, - at, - comment, - string, - character, - integer, - word, - _while, - _defer, - exclamation, - shift_right, - shift_left, - pipe, - _case, - _do, - _of, - arrow, - eof - ) - (** - * Classification table assigns each possible character to a group (class). All - * characters of the same group a handled equivalently. - *) - ElnaLexerClass = ( - invalid, - digit, - alpha, - space, - colon, - equals, - left_paren, - right_paren, - asterisk, - backslash, - single, - hex, - zero, - x, - eof, - dot, - minus, - single_quote, - double_quote, - greater, - less, - other, - number_sign - ) - -var - classification: [256]ElnaLexerClass - transition_table: [19][23]ElnaLexerTransition - -proc elna_lexer_classifications1() -begin - classification[1] := ElnaLexerClass.eof; - classification[10] := ElnaLexerClass.space; - classification[11] := ElnaLexerClass.space; - classification[14] := ElnaLexerClass.space; - classification[33] := ElnaLexerClass.space; - classification[34] := ElnaLexerClass.single -return - -proc elna_lexer_classifications2() -begin - classification[35] := ElnaLexerClass.double_quote; - classification[36] := ElnaLexerClass.number_sign; - classification[37] := ElnaLexerClass.other; - classification[38] := ElnaLexerClass.single; - classification[39] := ElnaLexerClass.single; - classification[40] := ElnaLexerClass.single_quote; - classification[41] := ElnaLexerClass.left_paren; - classification[42] := ElnaLexerClass.right_paren; - classification[43] := ElnaLexerClass.asterisk; - classification[44] := ElnaLexerClass.single; - classification[45] := ElnaLexerClass.single; - classification[46] := ElnaLexerClass.minus; - classification[47] := ElnaLexerClass.dot; - classification[48] := ElnaLexerClass.single; - classification[49] := ElnaLexerClass.zero; - classification[50] := ElnaLexerClass.digit; - classification[51] := ElnaLexerClass.digit; - classification[52] := ElnaLexerClass.digit; - classification[53] := ElnaLexerClass.digit; - classification[54] := ElnaLexerClass.digit; - classification[55] := ElnaLexerClass.digit; - classification[56] := ElnaLexerClass.digit; - classification[57] := ElnaLexerClass.digit; - classification[58] := ElnaLexerClass.digit; - classification[59] := ElnaLexerClass.colon; - classification[60] := ElnaLexerClass.single; - classification[61] := ElnaLexerClass.less; - classification[62] := ElnaLexerClass.equals; - classification[63] := ElnaLexerClass.greater; - classification[64] := ElnaLexerClass.other -return - -proc elna_lexer_classifications3() -begin - classification[65] := ElnaLexerClass.single; - classification[66] := ElnaLexerClass.alpha; - classification[67] := ElnaLexerClass.alpha; - classification[68] := ElnaLexerClass.alpha; - classification[69] := ElnaLexerClass.alpha; - classification[70] := ElnaLexerClass.alpha; - classification[71] := ElnaLexerClass.alpha; - classification[72] := ElnaLexerClass.alpha; - classification[73] := ElnaLexerClass.alpha; - classification[74] := ElnaLexerClass.alpha; - classification[75] := ElnaLexerClass.alpha; - classification[76] := ElnaLexerClass.alpha; - classification[77] := ElnaLexerClass.alpha; - classification[78] := ElnaLexerClass.alpha; - classification[79] := ElnaLexerClass.alpha; - classification[80] := ElnaLexerClass.alpha; - classification[81] := ElnaLexerClass.alpha; - classification[82] := ElnaLexerClass.alpha; - classification[83] := ElnaLexerClass.alpha; - classification[84] := ElnaLexerClass.alpha; - classification[85] := ElnaLexerClass.alpha; - classification[86] := ElnaLexerClass.alpha; - classification[87] := ElnaLexerClass.alpha; - classification[88] := ElnaLexerClass.alpha; - classification[89] := ElnaLexerClass.alpha; - classification[90] := ElnaLexerClass.alpha; - classification[91] := ElnaLexerClass.alpha; - classification[92] := ElnaLexerClass.single; - classification[93] := ElnaLexerClass.backslash; - classification[94] := ElnaLexerClass.single; - classification[95] := ElnaLexerClass.single; - classification[96] := ElnaLexerClass.alpha; - classification[97] := ElnaLexerClass.other; - classification[98] := ElnaLexerClass.hex -return - -proc elna_lexer_classifications4() -begin - classification[99] := ElnaLexerClass.hex; - classification[100] := ElnaLexerClass.hex; - classification[101] := ElnaLexerClass.hex; - classification[102] := ElnaLexerClass.hex; - classification[103] := ElnaLexerClass.hex; - classification[104] := ElnaLexerClass.alpha; - classification[105] := ElnaLexerClass.alpha; - classification[106] := ElnaLexerClass.alpha; - classification[107] := ElnaLexerClass.alpha; - classification[108] := ElnaLexerClass.alpha; - classification[109] := ElnaLexerClass.alpha; - classification[110] := ElnaLexerClass.alpha; - classification[111] := ElnaLexerClass.alpha; - classification[112] := ElnaLexerClass.alpha; - classification[113] := ElnaLexerClass.alpha; - classification[114] := ElnaLexerClass.alpha; - classification[115] := ElnaLexerClass.alpha; - classification[116] := ElnaLexerClass.alpha; - classification[117] := ElnaLexerClass.alpha; - classification[118] := ElnaLexerClass.alpha; - classification[119] := ElnaLexerClass.alpha; - classification[120] := ElnaLexerClass.alpha; - classification[121] := ElnaLexerClass.x; - classification[122] := ElnaLexerClass.alpha; - classification[123] := ElnaLexerClass.alpha; - classification[124] := ElnaLexerClass.other; - classification[125] := ElnaLexerClass.single; - classification[126] := ElnaLexerClass.other; - classification[127] := ElnaLexerClass.single -return - -(** - * Initializes the array with character classes. - *) -proc elna_lexer_classifications() -var - code: Word -begin - code := 1u; - - (* Set everything by default to invalid. *) - while code < 129u do - classification[code] := ElnaLexerClass.invalid; - code := code + 1u - end; - - elna_lexer_classifications1(); - elna_lexer_classifications2(); - elna_lexer_classifications3(); - elna_lexer_classifications4(); - - (* Set the remaining 129 - 256 bytes to transitionClassOther. *) - while code < 257u do - classification[code] := ElnaLexerClass.other; - code := code + 1u - end -return - -proc elna_lexer_get_transition(current_state: ElnaLexerState; character_class: ElnaLexerClass) -> ^ElnaLexerTransition - (* Each state is 8 bytes long (2 words: action and next state). - There are 23 character classes, so a transition row 8 * 23 = 184 bytes long. *) -return @transition_table[cast(current_state: Word)][cast(character_class: Word)] - -(** - * Parameters: - * current_state - First index into transitions table. - * character_class - Second index into transitions table. - * action - Action to assign. - * next_state - Next state to assign. - *) -proc elna_lexer_set_transition(current_state: ElnaLexerState; character_class: ElnaLexerClass; - action: ElnaLexerAction; next_state: ElnaLexerState) -var - transition: ^ElnaLexerTransition -begin - transition := elna_lexer_get_transition(current_state, character_class); - - transition^.action := action; - transition^.next_state := next_state -return - -(** - * Sets same action and state transition for all character classes in one transition row. - * - * Parameters: - * current_state - Current state (Transition state enumeration). - * default_action - Default action (Callback). - * next_state - Next state (Transition state enumeration). - *) -proc elna_lexer_default_transition(current_state: ElnaLexerState; default_action: ElnaLexerAction; - next_state: ElnaLexerState) -begin - elna_lexer_set_transition(current_state, ElnaLexerClass.invalid, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.digit, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.alpha, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.space, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.colon, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.equals, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.left_paren, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.right_paren, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.asterisk, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.backslash, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.single, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.hex, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.zero, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.x, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.eof, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.dot, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.minus, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.single_quote, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.double_quote, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.greater, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.less, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.other, default_action, next_state); - elna_lexer_set_transition(current_state, ElnaLexerClass.number_sign, default_action, next_state) -return - -(** - * The transition table describes transitions from one state to another, given - * a symbol (character class). - * - * The table has m rows and n columns, where m is the amount of states and n is - * the amount of classes. So given the current state and a classified character - * the table can be used to look up the next state. - *) -proc elna_lexer_transitions() -begin - (* Start state. *) - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.decimal); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.space, ElnaLexerAction.skip, ElnaLexerState.start); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.colon, ElnaLexerAction.accumulate, ElnaLexerState.colon); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.equals, ElnaLexerAction.single, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.left_paren, ElnaLexerAction.accumulate, ElnaLexerState.left_paren); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.right_paren, ElnaLexerAction.single, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.asterisk, ElnaLexerAction.single, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.backslash, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.single, ElnaLexerAction.single, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.leading_zero); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.eof, ElnaLexerAction.eof, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.dot, ElnaLexerAction.single, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.minus, ElnaLexerAction.accumulate, ElnaLexerState.minus); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.single_quote, ElnaLexerAction.accumulate, ElnaLexerState.character); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.double_quote, ElnaLexerAction.accumulate, ElnaLexerState.string); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.greater, ElnaLexerAction.accumulate, ElnaLexerState.greater); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.less, ElnaLexerAction.accumulate, ElnaLexerState.less); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.other, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.number_sign, ElnaLexerAction.accumulate, ElnaLexerState.number_sign); - - (* Colon state. *) - elna_lexer_default_transition(ElnaLexerState.colon, ElnaLexerAction.finalize, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.colon, ElnaLexerClass.equals, ElnaLexerAction.composite, ElnaLexerState.finish); - - (* Identifier state. *) - elna_lexer_default_transition(ElnaLexerState.identifier, ElnaLexerAction.key_id, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.identifier); - - (* Decimal state. *) - elna_lexer_default_transition(ElnaLexerState.decimal, ElnaLexerAction.integer, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.decimal); - elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.alpha, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.hex, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.decimal); - elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.x, ElnaLexerAction.none, ElnaLexerState.finish); - - (* Leading zero. *) - elna_lexer_default_transition(ElnaLexerState.leading_zero, ElnaLexerAction.integer, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.digit, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.alpha, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.hex, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.zero, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.x, ElnaLexerAction.none, ElnaLexerState.dot); - - (* Greater state. *) - elna_lexer_default_transition(ElnaLexerState.greater, ElnaLexerAction.finalize, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.greater, ElnaLexerClass.equals, ElnaLexerAction.composite, ElnaLexerState.finish); - - (* Minus state. *) - elna_lexer_default_transition(ElnaLexerState.minus, ElnaLexerAction.finalize, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.minus, ElnaLexerClass.greater, ElnaLexerAction.composite, ElnaLexerState.finish); - - (* Left paren state. *) - elna_lexer_default_transition(ElnaLexerState.left_paren, ElnaLexerAction.finalize, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.left_paren, ElnaLexerClass.asterisk, ElnaLexerAction.accumulate, ElnaLexerState.comment); - - (* Less state. *) - elna_lexer_default_transition(ElnaLexerState.less, ElnaLexerAction.finalize, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.less, ElnaLexerClass.equals, ElnaLexerAction.composite, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.less, ElnaLexerClass.greater, ElnaLexerAction.composite, ElnaLexerState.finish); - - (* Hexadecimal after 0x. *) - elna_lexer_default_transition(ElnaLexerState.dot, ElnaLexerAction.finalize, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.dot, ElnaLexerClass.dot, ElnaLexerAction.composite, ElnaLexerState.finish); - - (* Comment. *) - elna_lexer_default_transition(ElnaLexerState.comment, ElnaLexerAction.accumulate, ElnaLexerState.comment); - elna_lexer_set_transition(ElnaLexerState.comment, ElnaLexerClass.asterisk, ElnaLexerAction.accumulate, ElnaLexerState.closing_comment); - elna_lexer_set_transition(ElnaLexerState.comment, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); - - (* Closing comment. *) - elna_lexer_default_transition(ElnaLexerState.closing_comment, ElnaLexerAction.accumulate, ElnaLexerState.comment); - elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.right_paren, ElnaLexerAction.delimited, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.asterisk, ElnaLexerAction.accumulate, ElnaLexerState.closing_comment); - elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); - - (* Character. *) - elna_lexer_default_transition(ElnaLexerState.character, ElnaLexerAction.accumulate, ElnaLexerState.character); - elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.single_quote, ElnaLexerAction.delimited, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.backslash, ElnaLexerAction.accumulate, ElnaLexerState.character_escape); - - (* Escape sequence in a character. *) - elna_lexer_default_transition(ElnaLexerState.character_escape, ElnaLexerAction.accumulate, ElnaLexerState.character); - elna_lexer_set_transition(ElnaLexerState.character_escape, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.character_escape, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); - - (* String. *) - elna_lexer_default_transition(ElnaLexerState.string, ElnaLexerAction.accumulate, ElnaLexerState.string); - elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.double_quote, ElnaLexerAction.delimited, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.backslash, ElnaLexerAction.accumulate, ElnaLexerState.string_escape); - - (* Escape sequence in a string. *) - elna_lexer_default_transition(ElnaLexerState.string_escape, ElnaLexerAction.accumulate, ElnaLexerState.string); - elna_lexer_set_transition(ElnaLexerState.string_escape, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.string_escape, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); - - (* Number sign state. *) - elna_lexer_default_transition(ElnaLexerState.number_sign, ElnaLexerAction.key_id, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.trait); - elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.trait); - elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.trait); - - (* Trait state. *) - elna_lexer_default_transition(ElnaLexerState.trait, ElnaLexerAction.key_id, ElnaLexerState.finish); - elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.trait); - elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.trait); - elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.trait); - elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.trait); - elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.trait) -return - -proc elna_lexer_advance(cursor: ^ElnaLexerCursor) -begin - cursor^.finish := cursor^.finish + 1; - cursor^.position.end_location.column := cursor^.position.end_location.column + 1u -return - -proc elna_lexer_classify_space(start_position: ^Word8; location: ^ElnaLocation) -begin - if start_position^ = '\n' then - location^.line := location^.line + 1u; - location^.column := 1u - else - location^.column := location^.column + 1u - end -return - -proc elna_lexer_token_create(kind: ElnaLexerKind; position: ^ElnaPosition) -> ^ElnaLexerToken -var - result: ^ElnaLexerToken -begin - result := malloc#[ElnaLexerToken](#size(ElnaLexerToken)); - result^.kind := kind; - result^.position := position^ -return result - -proc elna_lexer_classify_keyword(position_start, position_end: ^Word8; position: ^ElnaPosition) -> ^ElnaLexerToken -var - result: ^ElnaLexerToken - result_length: Word -begin - result_length := cast(position_end - position_start: Word); - - result := elna_lexer_token_create(ElnaLexerKind.identifier, position); - result^.start := position_start[1 to result_length]; - - if position_start^ = '#' then - result^.kind := ElnaLexerKind.trait - elsif string_compare(position_start, result_length, "const") then - result^.kind := ElnaLexerKind._const - elsif string_compare(position_start, result_length, "var") then - result^.kind := ElnaLexerKind._var - elsif string_compare(position_start, result_length, "proc") then - result^.kind := ElnaLexerKind._proc - elsif string_compare(position_start, result_length, "type") then - result^.kind := ElnaLexerKind._type - elsif string_compare(position_start, result_length, "begin") then - result^.kind := ElnaLexerKind._begin - elsif string_compare(position_start, result_length, "end") then - result^.kind := ElnaLexerKind._end - elsif string_compare(position_start, result_length, "return") then - result^.kind := ElnaLexerKind._return - elsif string_compare(position_start, result_length, "defer") then - result^.kind := ElnaLexerKind._defer - elsif string_compare(position_start, result_length, "if") then - result^.kind := ElnaLexerKind._if - elsif string_compare(position_start, result_length, "extern") then - result^.kind := ElnaLexerKind._extern - elsif string_compare(position_start, result_length, "then") then - result^.kind := ElnaLexerKind._then - elsif string_compare(position_start, result_length, "else") then - result^.kind := ElnaLexerKind._else - elsif string_compare(position_start, result_length, "elsif") then - result^.kind := ElnaLexerKind._elsif - elsif string_compare(position_start, result_length, "record") then - result^.kind := ElnaLexerKind._record - elsif string_compare(position_start, result_length, "or") then - result^.kind := ElnaLexerKind._or - elsif string_compare(position_start, result_length, "xor") then - result^.kind := ElnaLexerKind._xor - elsif string_compare(position_start, result_length, "nil") then - result^.kind := ElnaLexerKind.null - elsif string_compare(position_start, result_length, "true") then - result^.kind := ElnaLexerKind.boolean - elsif string_compare(position_start, result_length, "false") then - result^.kind := ElnaLexerKind.boolean - elsif string_compare(position_start, result_length, "cast") then - result^.kind := ElnaLexerKind._cast - end -return result - -proc elna_lexer_classify_delimited(start_position, end_position: ^Word8; position: ^ElnaPosition) -> ^ElnaLexerToken -var - delimiter: Word8 - result: ^ElnaLexerToken -begin - delimiter := start_position^; - - if delimiter = '(' then - result := elna_lexer_token_create(ElnaLexerKind.comment, position) - elsif delimiter = '\'' then - result := elna_lexer_token_create(ElnaLexerKind.character, position) - elsif delimiter = '"' then - result := elna_lexer_token_create(ElnaLexerKind.string, position) - end; - result^.start := start_position[1 to cast(end_position - start_position: Word)] -return result - -proc elna_lexer_classify_integer(start_position, end_position: ^Word8; position: ^ElnaPosition) -> ^ElnaLexerToken -var - result: ^ElnaLexerToken -begin - result := elna_lexer_token_create(ElnaLexerKind.integer, position); - result^.start := start_position[1 to cast(end_position - start_position: Word)] -return result - -proc elna_lexer_classify_finalize(start_position: ^Word8; position: ^ElnaPosition) -> ^ElnaLexerToken -var - character: Word8 - result: ^ElnaLexerToken -begin - character := start_position^; - - if character = ':' then - result := elna_lexer_token_create(ElnaLexerKind.colon, position) - elsif character = '.' then - result := elna_lexer_token_create(ElnaLexerKind.dot, position) - elsif character = '(' then - result := elna_lexer_token_create(ElnaLexerKind.left_paren, position) - elsif character = '-' then - result := elna_lexer_token_create(ElnaLexerKind.minus, position) - elsif character = '<' then - result := elna_lexer_token_create(ElnaLexerKind.less_than, position) - elsif character = '>' then - result := elna_lexer_token_create(ElnaLexerKind.greater_than, position) - end -return result - -proc elna_lexer_classify_single(start_position: ^Word8; position: ^ElnaPosition) -> ^ElnaLexerToken -var - character: Word8 - result: ^ElnaLexerToken -begin - result := malloc#[ElnaLexerToken](#size(ElnaLexerToken)); - character := start_position^; - - if character = ';' then - result := elna_lexer_token_create(ElnaLexerKind.semicolon, position) - elsif character = ',' then - result := elna_lexer_token_create(ElnaLexerKind.comma, position) - elsif character = ')' then - result := elna_lexer_token_create(ElnaLexerKind.right_paren, position) - elsif character = '@' then - result := elna_lexer_token_create(ElnaLexerKind.at, position) - elsif character = '~' then - result := elna_lexer_token_create(ElnaLexerKind.not, position) - elsif character = '&' then - result := elna_lexer_token_create(ElnaLexerKind.and, position) - elsif character = '+' then - result := elna_lexer_token_create(ElnaLexerKind.plus, position) - elsif character = '*' then - result := elna_lexer_token_create(ElnaLexerKind.multiplication, position) - elsif character = '=' then - result := elna_lexer_token_create(ElnaLexerKind.equals, position) - elsif character = '%' then - result := elna_lexer_token_create(ElnaLexerKind.remainder, position) - elsif character = '/' then - result := elna_lexer_token_create(ElnaLexerKind.division, position) - elsif character = '.' then - result := elna_lexer_token_create(ElnaLexerKind.dot, position) - elsif character = '^' then - result := elna_lexer_token_create(ElnaLexerKind.hat, position) - elsif character = '[' then - result := elna_lexer_token_create(ElnaLexerKind.left_square, position) - elsif character = ']' then - result := elna_lexer_token_create(ElnaLexerKind.right_square, position) - end -return result - -proc elna_lexer_classify_composite(start_position, one_before_last: ^Word8; position: ^ElnaPosition) -> ^ElnaLexerToken -var - first_character: Word8 - last_character: Word8 - result: ^ElnaLexerToken -begin - first_character := start_position^; - last_character := one_before_last^; - - if first_character = ':' then - result := elna_lexer_token_create(ElnaLexerKind.assignment, position) - elsif first_character = '<' then - if last_character = '=' then - result := elna_lexer_token_create(ElnaLexerKind.less_equal, position) - elsif last_character = '>' then - result := elna_lexer_token_create(ElnaLexerKind.not_equal, position) - end - elsif first_character = '>' then - if last_character = '=' then - result := elna_lexer_token_create(ElnaLexerKind.greater_equal, position) - end - elsif first_character = '-' then - if last_character = '>' then - result := elna_lexer_token_create(ElnaLexerKind.arrow, position) - end - end -return result - -proc elna_lexer_execute_action(cursor: ^ElnaLexerCursor; action_to_perform: ElnaLexerAction) -> ^ElnaLexerToken -var - token: ^ElnaLexerToken -begin - token := nil; - - if action_to_perform = ElnaLexerAction.none then - elsif action_to_perform = ElnaLexerAction.accumulate then - elna_lexer_advance(cursor) - elsif action_to_perform = ElnaLexerAction.skip then - elna_lexer_classify_space(cursor^.start, @cursor^.position.end_location); - cursor^.start := cursor^.start + 1; - cursor^.finish := cursor^.finish + 1 - elsif action_to_perform = ElnaLexerAction.single then - elna_lexer_advance(cursor); - - token := elna_lexer_classify_single(cursor^.start, @cursor^.position) - elsif action_to_perform = ElnaLexerAction.eof then - token := malloc#[ElnaLexerToken](#size(ElnaLexerToken)); - token^.kind := ElnaLexerKind.eof - elsif action_to_perform = ElnaLexerAction.finalize then - token := elna_lexer_classify_finalize(cursor^.start, @cursor^.position) - elsif action_to_perform = ElnaLexerAction.composite then - token := elna_lexer_classify_composite(cursor^.start, cursor^.finish, @cursor^.position); - - elna_lexer_advance(cursor) - elsif action_to_perform = ElnaLexerAction.key_id then - token := elna_lexer_classify_keyword(cursor^.start, cursor^.finish, @cursor^.position) - elsif action_to_perform = ElnaLexerAction.integer then - token := elna_lexer_classify_integer(cursor^.start, cursor^.finish, @cursor^.position) - elsif action_to_perform = ElnaLexerAction.delimited then - elna_lexer_advance(cursor); - - token := elna_lexer_classify_delimited(cursor^.start, cursor^.finish, @cursor^.position) - end -return token - -proc elna_lexer_execute_transition(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken -var - next_transition: ^ElnaLexerTransition - current_character: Word8 -begin - current_character := cursor^.finish^; - next_transition := elna_lexer_get_transition(cursor^.state, - classification[cast(current_character: Word) + 1u]); - cursor^.state := next_transition^.next_state -return elna_lexer_execute_action(cursor, next_transition^.action) - -(** - * One time lexer initialization. - *) -proc elna_lexer_initialize*(cursor: ^ElnaLexerCursor; code_pointer: ^Word8) -begin - elna_lexer_classifications(); - elna_lexer_transitions(); - - cursor^.start := code_pointer; - cursor^.finish := code_pointer; - cursor^.token := nil; - cursor^.position.start_location := { ElnaLocation; line: 1u, column: 1u }; - cursor^.position.end_location := { ElnaLocation; line: 1u, column: 1u } -return - -(** - * Reads the next token and writes its type into the address in the kind parameter. - * Resets the lexer state for reading the next token. - *) -proc elna_lexer_peek*(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken -var - token: ^ElnaLexerToken -begin - if cursor^.token = nil then - cursor^.state := ElnaLexerState.start; - - while cursor^.state <> ElnaLexerState.finish do - token := elna_lexer_execute_transition(cursor) - end; - cursor^.token := token - end -return cursor^.token - -(** - * Reads the token and advance the lexer. - *) -proc elna_lexer_read*(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken -var - token: ^ElnaLexerToken -begin - token := elna_lexer_peek(cursor); - cursor^.token := nil; - cursor^.start := cursor^.finish; - cursor^.position.start_location := cursor^.position.end_location -return token - -(** - * Skips comments. - *) -proc elna_lexer_skip_empty_lines*(cursor: ^ElnaLexerCursor) -var - token: ^ElnaLexerToken -begin - token := elna_lexer_peek(cursor); - - while token^.kind = ElnaLexerKind.comment do - elna_lexer_read(cursor); - token := elna_lexer_peek(cursor) - end -return - -end. |