Allow any word type for shift operators

1 files changed, 9 insertions, 9 deletions

diff --git a/source/main.elna b/source/main.elna
index fb58766..2099cb3 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer
 
 var
   global_cursor: ElnaLexerCursor
-  sample_source_text := "Here.are a few 'tokens' or if\0"
+  sample_source_text := "Here.are a few (* documentation block *) or if -- End line comment\0"
   token: ^ElnaLexerToken
 
 begin
@@ -17,14 +17,14 @@ begin
     position: 1u
   };
 
-  repeat
-    token := elna_lexer_lex(@global_cursor);
+  token := elna_lexer_lex(@global_cursor);
+  elna_lexer_token_dump(token, @global_cursor);
 
-    printf("Token: %.*s, length: %lu, final state: %i, position: %lu, kind: %i\n\0".ptr,
-        global_cursor.cursor - global_cursor.start, global_cursor.start,
-        global_cursor.cursor - global_cursor.start, global_cursor.state,
-        global_cursor.position, token^.kind);
+  while token^.kind <> ElnaLexerTokenKind.eof do
+    elna_lexer_token_destroy(token);
 
-    elna_lexer_token_destroy(token)
-  until global_cursor.state = ElnaLexerState.eof
+    token := elna_lexer_lex(@global_cursor);
+    elna_lexer_token_dump(token, @global_cursor)
+  end;
+  elna_lexer_token_destroy(token)
 end.