(* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. *) import cstdio, common, cstdlib type ElnaLexerState = ( start, colon, identifier, decimal, leading_zero, greater, minus, left_paren, less, dot, comment, closing_comment, character, character_escape, string, string_escape, number_sign, trait, finish ) ElnaLexerToken* = record kind: ElnaLexerKind; start: []const Char; (* DEPRECATED *) position: ElnaPosition end ElnaLexerBooleanToken* = record(ElnaLexerToken) value: Bool end ElnaLexerCharacterToken* = record(ElnaLexerToken) value: Char end ElnaLexerStringToken* = record(ElnaLexerToken) value: []const Char end ElnaLexerIntegerToken* = record(ElnaLexerToken) value: Int end TransitionAction = proc(lexer: ^Lexer; token: ^ElnaLexerToken) (* DEPRECATED should be replaced with TransitionAction (but keeping the type name). *) ElnaLexerAction = (none, accumulate, skip, single, eof, finalize, composite, key_id, integer, delimited) ElnaLexerTransition = record action: ElnaLexerAction; next_state: ElnaLexerState end ElnaLexerCursor = record state: ElnaLexerState; start: ^Char; finish: ^Char; token: ^ElnaLexerToken; position: ElnaPosition end BufferPosition* = record iterator: ^Char; location: ElnaLocation end Lexer* = record input: ^FILE; buffer: ^Char; size: Word; length: Word; start: BufferPosition; current: BufferPosition end ElnaLexerKind* = ( identifier, _const, _var, _proc, _type, _begin, _end, _if, _then, _else, _elsif, _extern, _record, boolean, null, and, _or, _xor, not, _return, _cast, trait, left_paren, right_paren, left_square, right_square, left_brace, right_brace, greater_equal, less_equal, greater_than, less_than, not_equal, equals, semicolon, dot, comma, plus, _import, minus, multiplication, division, remainder, assignment, colon, hat, at, comment, string, character, integer, word, _while, _defer, exclamation, shift_right, shift_left, pipe, _case, _do, _of, eof ) (** * Classification table assigns each possible character to a group (class). All * characters of the same group a handled equivalently. *) ElnaLexerClass = ( invalid, digit, alpha, space, colon, equals, left_paren, right_paren, asterisk, backslash, single, hex, zero, x, eof, dot, minus, single_quote, double_quote, greater, less, other, number_sign ) var classification: [256]ElnaLexerClass transition_table: [19][23]ElnaLexerTransition proc elna_lexer_classifications1() begin classification[1] := ElnaLexerClass.eof; classification[10] := ElnaLexerClass.space; classification[11] := ElnaLexerClass.space; classification[14] := ElnaLexerClass.space; classification[33] := ElnaLexerClass.space; classification[34] := ElnaLexerClass.single return proc elna_lexer_classifications2() begin classification[35] := ElnaLexerClass.double_quote; classification[36] := ElnaLexerClass.number_sign; classification[37] := ElnaLexerClass.other; classification[38] := ElnaLexerClass.single; classification[39] := ElnaLexerClass.single; classification[40] := ElnaLexerClass.single_quote; classification[41] := ElnaLexerClass.left_paren; classification[42] := ElnaLexerClass.right_paren; classification[43] := ElnaLexerClass.asterisk; classification[44] := ElnaLexerClass.single; classification[45] := ElnaLexerClass.single; classification[46] := ElnaLexerClass.minus; classification[47] := ElnaLexerClass.dot; classification[48] := ElnaLexerClass.single; classification[49] := ElnaLexerClass.zero; classification[50] := ElnaLexerClass.digit; classification[51] := ElnaLexerClass.digit; classification[52] := ElnaLexerClass.digit; classification[53] := ElnaLexerClass.digit; classification[54] := ElnaLexerClass.digit; classification[55] := ElnaLexerClass.digit; classification[56] := ElnaLexerClass.digit; classification[57] := ElnaLexerClass.digit; classification[58] := ElnaLexerClass.digit; classification[59] := ElnaLexerClass.colon; classification[60] := ElnaLexerClass.single; classification[61] := ElnaLexerClass.less; classification[62] := ElnaLexerClass.equals; classification[63] := ElnaLexerClass.greater; classification[64] := ElnaLexerClass.other return proc elna_lexer_classifications3() begin classification[65] := ElnaLexerClass.single; classification[66] := ElnaLexerClass.alpha; classification[67] := ElnaLexerClass.alpha; classification[68] := ElnaLexerClass.alpha; classification[69] := ElnaLexerClass.alpha; classification[70] := ElnaLexerClass.alpha; classification[71] := ElnaLexerClass.alpha; classification[72] := ElnaLexerClass.alpha; classification[73] := ElnaLexerClass.alpha; classification[74] := ElnaLexerClass.alpha; classification[75] := ElnaLexerClass.alpha; classification[76] := ElnaLexerClass.alpha; classification[77] := ElnaLexerClass.alpha; classification[78] := ElnaLexerClass.alpha; classification[79] := ElnaLexerClass.alpha; classification[80] := ElnaLexerClass.alpha; classification[81] := ElnaLexerClass.alpha; classification[82] := ElnaLexerClass.alpha; classification[83] := ElnaLexerClass.alpha; classification[84] := ElnaLexerClass.alpha; classification[85] := ElnaLexerClass.alpha; classification[86] := ElnaLexerClass.alpha; classification[87] := ElnaLexerClass.alpha; classification[88] := ElnaLexerClass.alpha; classification[89] := ElnaLexerClass.alpha; classification[90] := ElnaLexerClass.alpha; classification[91] := ElnaLexerClass.alpha; classification[92] := ElnaLexerClass.single; classification[93] := ElnaLexerClass.backslash; classification[94] := ElnaLexerClass.single; classification[95] := ElnaLexerClass.single; classification[96] := ElnaLexerClass.alpha; classification[97] := ElnaLexerClass.other; classification[98] := ElnaLexerClass.hex return proc elna_lexer_classifications4() begin classification[99] := ElnaLexerClass.hex; classification[100] := ElnaLexerClass.hex; classification[101] := ElnaLexerClass.hex; classification[102] := ElnaLexerClass.hex; classification[103] := ElnaLexerClass.hex; classification[104] := ElnaLexerClass.alpha; classification[105] := ElnaLexerClass.alpha; classification[106] := ElnaLexerClass.alpha; classification[107] := ElnaLexerClass.alpha; classification[108] := ElnaLexerClass.alpha; classification[109] := ElnaLexerClass.alpha; classification[110] := ElnaLexerClass.alpha; classification[111] := ElnaLexerClass.alpha; classification[112] := ElnaLexerClass.alpha; classification[113] := ElnaLexerClass.alpha; classification[114] := ElnaLexerClass.alpha; classification[115] := ElnaLexerClass.alpha; classification[116] := ElnaLexerClass.alpha; classification[117] := ElnaLexerClass.alpha; classification[118] := ElnaLexerClass.alpha; classification[119] := ElnaLexerClass.alpha; classification[120] := ElnaLexerClass.alpha; classification[121] := ElnaLexerClass.x; classification[122] := ElnaLexerClass.alpha; classification[123] := ElnaLexerClass.alpha; classification[124] := ElnaLexerClass.other; classification[125] := ElnaLexerClass.single; classification[126] := ElnaLexerClass.other; classification[127] := ElnaLexerClass.single return (** * Initializes the array with character classes. *) proc elna_lexer_classifications() var code: Word begin code := 1u; (* Set everything by default to invalid. *) while code < 129u do classification[code] := ElnaLexerClass.invalid; code := code + 1u end; elna_lexer_classifications1(); elna_lexer_classifications2(); elna_lexer_classifications3(); elna_lexer_classifications4(); (* Set the remaining 129 - 256 bytes to transitionClassOther. *) while code < 257u do classification[code] := ElnaLexerClass.other; code := code + 1u end return proc elna_lexer_get_transition(current_state: ElnaLexerState; character_class: ElnaLexerClass): ^ElnaLexerTransition (* Each state is 8 bytes long (2 words: action and next state). There are 23 character classes, so a transition row 8 * 23 = 184 bytes long. *) return @transition_table[cast(current_state: Word)][cast(character_class: Word)] (** * Parameters: * current_state - First index into transitions table. * character_class - Second index into transitions table. * action - Action to assign. * next_state - Next state to assign. *) proc elna_lexer_set_transition(current_state: ElnaLexerState; character_class: ElnaLexerClass; action: ElnaLexerAction; next_state: ElnaLexerState) var transition: ^ElnaLexerTransition begin transition := elna_lexer_get_transition(current_state, character_class); transition^.action := action; transition^.next_state := next_state return (** * Sets same action and state transition for all character classes in one transition row. * * Parameters: * current_state - Current state (Transition state enumeration). * default_action - Default action (Callback). * next_state - Next state (Transition state enumeration). *) proc elna_lexer_default_transition(current_state: ElnaLexerState; default_action: ElnaLexerAction; next_state: ElnaLexerState) begin elna_lexer_set_transition(current_state, ElnaLexerClass.invalid, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.digit, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.alpha, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.space, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.colon, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.equals, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.left_paren, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.right_paren, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.asterisk, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.backslash, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.single, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.hex, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.zero, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.x, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.eof, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.dot, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.minus, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.single_quote, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.double_quote, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.greater, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.less, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.other, default_action, next_state); elna_lexer_set_transition(current_state, ElnaLexerClass.number_sign, default_action, next_state) return (** * The transition table describes transitions from one state to another, given * a symbol (character class). * * The table has m rows and n columns, where m is the amount of states and n is * the amount of classes. So given the current state and a classified character * the table can be used to look up the next state. *) proc elna_lexer_transitions() begin (* Start state. *) elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.decimal); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.space, ElnaLexerAction.skip, ElnaLexerState.start); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.colon, ElnaLexerAction.accumulate, ElnaLexerState.colon); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.equals, ElnaLexerAction.single, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.left_paren, ElnaLexerAction.accumulate, ElnaLexerState.left_paren); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.right_paren, ElnaLexerAction.single, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.asterisk, ElnaLexerAction.single, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.backslash, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.single, ElnaLexerAction.single, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.leading_zero); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.eof, ElnaLexerAction.eof, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.dot, ElnaLexerAction.single, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.minus, ElnaLexerAction.accumulate, ElnaLexerState.minus); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.single_quote, ElnaLexerAction.accumulate, ElnaLexerState.character); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.double_quote, ElnaLexerAction.accumulate, ElnaLexerState.string); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.greater, ElnaLexerAction.accumulate, ElnaLexerState.greater); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.less, ElnaLexerAction.accumulate, ElnaLexerState.less); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.other, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.number_sign, ElnaLexerAction.accumulate, ElnaLexerState.number_sign); (* Colon state. *) elna_lexer_default_transition(ElnaLexerState.colon, ElnaLexerAction.finalize, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.colon, ElnaLexerClass.equals, ElnaLexerAction.composite, ElnaLexerState.finish); (* Identifier state. *) elna_lexer_default_transition(ElnaLexerState.identifier, ElnaLexerAction.key_id, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.identifier); elna_lexer_set_transition(ElnaLexerState.identifier, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.identifier); (* Decimal state. *) elna_lexer_default_transition(ElnaLexerState.decimal, ElnaLexerAction.integer, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.decimal); elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.alpha, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.hex, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.decimal); elna_lexer_set_transition(ElnaLexerState.decimal, ElnaLexerClass.x, ElnaLexerAction.none, ElnaLexerState.finish); (* Leading zero. *) elna_lexer_default_transition(ElnaLexerState.leading_zero, ElnaLexerAction.integer, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.digit, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.alpha, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.hex, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.zero, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.x, ElnaLexerAction.none, ElnaLexerState.dot); (* Greater state. *) elna_lexer_default_transition(ElnaLexerState.greater, ElnaLexerAction.finalize, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.greater, ElnaLexerClass.equals, ElnaLexerAction.composite, ElnaLexerState.finish); (* Minus state. *) elna_lexer_default_transition(ElnaLexerState.minus, ElnaLexerAction.finalize, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.minus, ElnaLexerClass.greater, ElnaLexerAction.composite, ElnaLexerState.finish); (* Left paren state. *) elna_lexer_default_transition(ElnaLexerState.left_paren, ElnaLexerAction.finalize, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.left_paren, ElnaLexerClass.asterisk, ElnaLexerAction.accumulate, ElnaLexerState.comment); (* Less state. *) elna_lexer_default_transition(ElnaLexerState.less, ElnaLexerAction.finalize, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.less, ElnaLexerClass.equals, ElnaLexerAction.composite, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.less, ElnaLexerClass.greater, ElnaLexerAction.composite, ElnaLexerState.finish); (* Hexadecimal after 0x. *) elna_lexer_default_transition(ElnaLexerState.dot, ElnaLexerAction.finalize, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.dot, ElnaLexerClass.dot, ElnaLexerAction.composite, ElnaLexerState.finish); (* Comment. *) elna_lexer_default_transition(ElnaLexerState.comment, ElnaLexerAction.accumulate, ElnaLexerState.comment); elna_lexer_set_transition(ElnaLexerState.comment, ElnaLexerClass.asterisk, ElnaLexerAction.accumulate, ElnaLexerState.closing_comment); elna_lexer_set_transition(ElnaLexerState.comment, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); (* Closing comment. *) elna_lexer_default_transition(ElnaLexerState.closing_comment, ElnaLexerAction.accumulate, ElnaLexerState.comment); elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.right_paren, ElnaLexerAction.delimited, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.asterisk, ElnaLexerAction.accumulate, ElnaLexerState.closing_comment); elna_lexer_set_transition(ElnaLexerState.closing_comment, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); (* Character. *) elna_lexer_default_transition(ElnaLexerState.character, ElnaLexerAction.accumulate, ElnaLexerState.character); elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.single_quote, ElnaLexerAction.delimited, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.character, ElnaLexerClass.backslash, ElnaLexerAction.accumulate, ElnaLexerState.character_escape); (* Escape sequence in a character. *) elna_lexer_default_transition(ElnaLexerState.character_escape, ElnaLexerAction.accumulate, ElnaLexerState.character); elna_lexer_set_transition(ElnaLexerState.character_escape, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.character_escape, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); (* String. *) elna_lexer_default_transition(ElnaLexerState.string, ElnaLexerAction.accumulate, ElnaLexerState.string); elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.double_quote, ElnaLexerAction.delimited, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.backslash, ElnaLexerAction.accumulate, ElnaLexerState.string_escape); (* Escape sequence in a string. *) elna_lexer_default_transition(ElnaLexerState.string_escape, ElnaLexerAction.accumulate, ElnaLexerState.string); elna_lexer_set_transition(ElnaLexerState.string_escape, ElnaLexerClass.invalid, ElnaLexerAction.none, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.string_escape, ElnaLexerClass.eof, ElnaLexerAction.none, ElnaLexerState.finish); (* Number sign state. *) elna_lexer_default_transition(ElnaLexerState.number_sign, ElnaLexerAction.key_id, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.trait); elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.trait); elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.trait); (* Trait state. *) elna_lexer_default_transition(ElnaLexerState.trait, ElnaLexerAction.key_id, ElnaLexerState.finish); elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.digit, ElnaLexerAction.accumulate, ElnaLexerState.trait); elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.alpha, ElnaLexerAction.accumulate, ElnaLexerState.trait); elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.hex, ElnaLexerAction.accumulate, ElnaLexerState.trait); elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.zero, ElnaLexerAction.accumulate, ElnaLexerState.trait); elna_lexer_set_transition(ElnaLexerState.trait, ElnaLexerClass.x, ElnaLexerAction.accumulate, ElnaLexerState.trait) return proc elna_lexer_advance(cursor: ^ElnaLexerCursor) begin cursor^.finish := cursor^.finish + 1; cursor^.position.end_location.column := cursor^.position.end_location.column + 1u return proc elna_lexer_classify_space(start_position: ^Char; location: ^ElnaLocation) begin if start_position^ = '\n' then location^.line := location^.line + 1u; location^.column := 1u else location^.column := location^.column + 1u end return proc elna_lexer_token_create(kind: ElnaLexerKind; position: ^ElnaPosition): ^ElnaLexerToken var result: ^ElnaLexerToken begin result := malloc(#size(ElnaLexerToken)); result^.kind := kind; result^.position := position^ return result proc elna_lexer_classify_keyword(position_start, position_end: ^Char; position: ^ElnaPosition): ^ElnaLexerToken var result: ^ElnaLexerToken result_length: Word begin result_length := cast(position_end - position_start: Word); result := elna_lexer_token_create(ElnaLexerKind.identifier, position); result^.start := position_start[1 to result_length]; if position_start^ = '#' then result^.kind := ElnaLexerKind.trait elsif string_compare(position_start, result_length, "const") then result^.kind := ElnaLexerKind._const elsif string_compare(position_start, result_length, "var") then result^.kind := ElnaLexerKind._var elsif string_compare(position_start, result_length, "proc") then result^.kind := ElnaLexerKind._proc elsif string_compare(position_start, result_length, "type") then result^.kind := ElnaLexerKind._type elsif string_compare(position_start, result_length, "begin") then result^.kind := ElnaLexerKind._begin elsif string_compare(position_start, result_length, "end") then result^.kind := ElnaLexerKind._end elsif string_compare(position_start, result_length, "return") then result^.kind := ElnaLexerKind._return elsif string_compare(position_start, result_length, "defer") then result^.kind := ElnaLexerKind._defer elsif string_compare(position_start, result_length, "if") then result^.kind := ElnaLexerKind._if elsif string_compare(position_start, result_length, "extern") then result^.kind := ElnaLexerKind._extern elsif string_compare(position_start, result_length, "then") then result^.kind := ElnaLexerKind._then elsif string_compare(position_start, result_length, "else") then result^.kind := ElnaLexerKind._else elsif string_compare(position_start, result_length, "elsif") then result^.kind := ElnaLexerKind._elsif elsif string_compare(position_start, result_length, "record") then result^.kind := ElnaLexerKind._record elsif string_compare(position_start, result_length, "or") then result^.kind := ElnaLexerKind._or elsif string_compare(position_start, result_length, "xor") then result^.kind := ElnaLexerKind._xor elsif string_compare(position_start, result_length, "nil") then result^.kind := ElnaLexerKind.null elsif string_compare(position_start, result_length, "true") then result^.kind := ElnaLexerKind.boolean elsif string_compare(position_start, result_length, "false") then result^.kind := ElnaLexerKind.boolean elsif string_compare(position_start, result_length, "cast") then result^.kind := ElnaLexerKind._cast end return result proc elna_lexer_classify_delimited(start_position, end_position: ^Char; position: ^ElnaPosition): ^ElnaLexerToken var delimiter: Char result: ^ElnaLexerToken begin delimiter := start_position^; if delimiter = '(' then result := elna_lexer_token_create(ElnaLexerKind.comment, position) elsif delimiter = '\'' then result := elna_lexer_token_create(ElnaLexerKind.character, position) elsif delimiter = '"' then result := elna_lexer_token_create(ElnaLexerKind.string, position) end; result^.start := start_position[1 to cast(end_position - start_position: Word)] return result proc elna_lexer_classify_integer(start_position, end_position: ^Char; position: ^ElnaPosition): ^ElnaLexerToken var result: ^ElnaLexerToken begin result := elna_lexer_token_create(ElnaLexerKind.integer, position); result^.start := start_position[1 to cast(end_position - start_position: Word)] return result proc elna_lexer_classify_finalize(start_position: ^Char; position: ^ElnaPosition): ^ElnaLexerToken var character: Char result: ^ElnaLexerToken begin character := start_position^; if character = ':' then result := elna_lexer_token_create(ElnaLexerKind.colon, position) elsif character = '.' then result := elna_lexer_token_create(ElnaLexerKind.dot, position) elsif character = '(' then result := elna_lexer_token_create(ElnaLexerKind.left_paren, position) elsif character = '-' then result := elna_lexer_token_create(ElnaLexerKind.minus, position) elsif character = '<' then result := elna_lexer_token_create(ElnaLexerKind.less_than, position) elsif character = '>' then result := elna_lexer_token_create(ElnaLexerKind.greater_than, position) end return result proc elna_lexer_classify_single(start_position: ^Char; position: ^ElnaPosition): ^ElnaLexerToken var character: Char result: ^ElnaLexerToken begin result := malloc(#size(ElnaLexerToken)); character := start_position^; if character = ';' then result := elna_lexer_token_create(ElnaLexerKind.semicolon, position) elsif character = ',' then result := elna_lexer_token_create(ElnaLexerKind.comma, position) elsif character = ')' then result := elna_lexer_token_create(ElnaLexerKind.right_paren, position) elsif character = '@' then result := elna_lexer_token_create(ElnaLexerKind.at, position) elsif character = '~' then result := elna_lexer_token_create(ElnaLexerKind.not, position) elsif character = '&' then result := elna_lexer_token_create(ElnaLexerKind.and, position) elsif character = '+' then result := elna_lexer_token_create(ElnaLexerKind.plus, position) elsif character = '*' then result := elna_lexer_token_create(ElnaLexerKind.multiplication, position) elsif character = '=' then result := elna_lexer_token_create(ElnaLexerKind.equals, position) elsif character = '%' then result := elna_lexer_token_create(ElnaLexerKind.remainder, position) elsif character = '/' then result := elna_lexer_token_create(ElnaLexerKind.division, position) elsif character = '.' then result := elna_lexer_token_create(ElnaLexerKind.dot, position) elsif character = '^' then result := elna_lexer_token_create(ElnaLexerKind.hat, position) elsif character = '[' then result := elna_lexer_token_create(ElnaLexerKind.left_square, position) elsif character = ']' then result := elna_lexer_token_create(ElnaLexerKind.right_square, position) end return result proc elna_lexer_classify_composite(start_position, one_before_last: ^Char; position: ^ElnaPosition): ^ElnaLexerToken var first_character: Char last_character: Char result: ^ElnaLexerToken begin first_character := start_position^; last_character := one_before_last^; if first_character = ':' then result := elna_lexer_token_create(ElnaLexerKind.assignment, position) elsif first_character = '<' then if last_character = '=' then result := elna_lexer_token_create(ElnaLexerKind.less_equal, position) elsif last_character = '>' then result := elna_lexer_token_create(ElnaLexerKind.not_equal, position) end elsif first_character = '>' then if last_character = '=' then result := elna_lexer_token_create(ElnaLexerKind.greater_equal, position) end end return result proc elna_lexer_execute_action(cursor: ^ElnaLexerCursor; action_to_perform: ElnaLexerAction): ^ElnaLexerToken var token: ^ElnaLexerToken begin token := nil; if action_to_perform = ElnaLexerAction.none then elsif action_to_perform = ElnaLexerAction.accumulate then elna_lexer_advance(cursor) elsif action_to_perform = ElnaLexerAction.skip then elna_lexer_classify_space(cursor^.start, @cursor^.position.end_location); cursor^.start := cursor^.start + 1; cursor^.finish := cursor^.finish + 1 elsif action_to_perform = ElnaLexerAction.single then elna_lexer_advance(cursor); token := elna_lexer_classify_single(cursor^.start, @cursor^.position) elsif action_to_perform = ElnaLexerAction.eof then token := malloc(#size(ElnaLexerToken)); token^.kind := ElnaLexerKind.eof elsif action_to_perform = ElnaLexerAction.finalize then token := elna_lexer_classify_finalize(cursor^.start, @cursor^.position) elsif action_to_perform = ElnaLexerAction.composite then token := elna_lexer_classify_composite(cursor^.start, cursor^.finish, @cursor^.position); elna_lexer_advance(cursor) elsif action_to_perform = ElnaLexerAction.key_id then token := elna_lexer_classify_keyword(cursor^.start, cursor^.finish, @cursor^.position) elsif action_to_perform = ElnaLexerAction.integer then token := elna_lexer_classify_integer(cursor^.start, cursor^.finish, @cursor^.position) elsif action_to_perform = ElnaLexerAction.delimited then elna_lexer_advance(cursor); token := elna_lexer_classify_delimited(cursor^.start, cursor^.finish, @cursor^.position) end return token proc elna_lexer_execute_transition(cursor: ^ElnaLexerCursor): ^ElnaLexerToken var next_transition: ^ElnaLexerTransition current_character: Char begin current_character := cursor^.finish^; next_transition := elna_lexer_get_transition(cursor^.state, classification[cast(current_character: Word) + 1u]); cursor^.state := next_transition^.next_state return elna_lexer_execute_action(cursor, next_transition^.action) (** * One time lexer initialization. *) proc elna_lexer_initialize(cursor: ^ElnaLexerCursor; code_pointer: ^Char) begin elna_lexer_classifications(); elna_lexer_transitions(); cursor^.start := code_pointer; cursor^.finish := code_pointer; cursor^.token := nil; cursor^.position.start_location := ElnaLocation{ line: 1u, column: 1u }; cursor^.position.end_location := ElnaLocation{ line: 1u, column: 1u } return (** * Reads the next token and writes its type into the address in the kind parameter. * Resets the lexer state for reading the next token. *) proc elna_lexer_peek(cursor: ^ElnaLexerCursor): ^ElnaLexerToken var token: ^ElnaLexerToken begin if cursor^.token = nil then cursor^.state := ElnaLexerState.start; while cursor^.state <> ElnaLexerState.finish do token := elna_lexer_execute_transition(cursor) end; cursor^.token := token end return cursor^.token (** * Reads the token and advance the lexer. *) proc elna_lexer_read(cursor: ^ElnaLexerCursor): ^ElnaLexerToken var token: ^ElnaLexerToken begin token := elna_lexer_peek(cursor); cursor^.token := nil; cursor^.start := cursor^.finish; cursor^.position.start_location := cursor^.position.end_location return token (** * Skips comments. *) proc elna_lexer_skip_empty_lines(cursor: ^ElnaLexerCursor) var token: ^ElnaLexerToken begin token := elna_lexer_peek(cursor); while token^.kind = ElnaLexerKind.comment do elna_lexer_read(cursor); token := elna_lexer_peek(cursor) end return end.