1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
(* This Source Code Form is subject to the terms of the Mozilla Public License,
v. 2.0. If a copy of the MPL was not distributed with this file, You can
obtain one at https://mozilla.org/MPL/2.0/. *)
program()
import sys.posix.stdio, elna.lexer
var
global_cursor: ElnaLexerCursor
sample_source_text := "Here.are a few 'tokens' or if\0"
token: ^ElnaLexerToken
begin
elna_lexer_transitions();
global_cursor := { ElnaLexerCursor;
start: sample_source_text.ptr,
cursor: sample_source_text.ptr,
position: 1u
};
repeat
token := elna_lexer_lex(@global_cursor);
printf("Token: %.*s, length: %lu, final state: %i, position: %lu, kind: %i\n\0".ptr,
global_cursor.cursor - global_cursor.start, global_cursor.start,
global_cursor.cursor - global_cursor.start, global_cursor.state,
global_cursor.position, token^.kind)
until global_cursor.state = ElnaLexerState.eof
end.
|
