Fix slice comparison by contents

7 files changed, 312 insertions, 50 deletions

diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna
index 7fa489a..64fcc22 100644
--- a/source/elna/lexer.elna
+++ b/source/elna/lexer.elna
@@ -5,7 +5,7 @@
 (* Lexical analysis. *)
 module
 
-import std, elna.source
+import std, sys.posix.stdlib, sys.posix.string, elna.source
 
 type
   (*
@@ -68,12 +68,47 @@ type
     (* Cannot appear in the source code. *)
     invalid
   )
-  (* Lexer state. *)
-  ElnaLexerState = (
+  (*
+    The order of the states in the enumeration is significant.
+
+    ElnaLexerState.dead is 0, so that the transition_table can be
+    zero-initialized with the dead state by default.
+
+    [ElnaLexerState.start, ElnaLexerState.identifier) are non-accepting states.
+
+    [ElnaLexerState.identifier, ElnaLexerState.stop) are accepting states, but
+    not final states.
+
+    Terminal states are at the end of the enumeration beginning with
+    ElnaLexerState.stop. transition_table doesn't allocate any space for them.
+  *)
+  ElnaLexerState* = (
     (* A non-accepting state from which no final state can ever be reached. *)
-    dead,
+    dead = 0,
     (* Start state. *)
     start,
+    (* Inside a docblock (after eating opening paren with asterisk). *)
+    docblock,
+    (* Seen a period after a number, possible floating point number. *)
+    dot_after,
+    (* Seen an asterisk inside a docblock. *)
+    closing_docblock,
+    (* Optional plus or minus sign following the exponent. *)
+    exponent_sign,
+    (* Entering UTF-32 character literal. *)
+    wide,
+    (* Seen an escape sequence in a UTF-32 character literal. *)
+    wide_escape,
+    (* Entering UTF-8 character literal. *)
+    narrow,
+    (* Seen an escape sequence in a UTF-8 character literal. *)
+    narrow_escape,
+    (* Entering string literal. *)
+    string,
+    (* Seen an escape sequence in a string literal. *)
+    string_escape,
+    (* Opening a compile-time feature starting with a #. *)
+    number_sign,
     (* Entering identifier or keyword. *)
     identifier,
     (* Skipping whitespaces. *)
@@ -88,42 +123,20 @@ type
     greater,
     (* Entering comparison operator starting with "less than". *)
     less,
-    (* Binary or hexadecimal word. *)
+    (* Numbers prefixed by 0 followed by a character (0b, 0x). *)
     leading_zero,
     (* Entering integer part of a number. *)
     integer,
     (* Inside a comment (after eating double minus, --). *)
     comment,
-    (* Inside a docblock (after eating opening paren with asterisk). *)
-    docblock,
-    (* Seen an asterisk inside a docblock. *)
-    closing_docblock,
     (* Tail is the last part of the number containing digits and an optional type suffix. *)
     number_tail,
-    (* Seen a period after a number, possible floating point number. *)
-    dot_after,
     (* Entering floating point number fraction part after a decimal point. *)
     fraction,
     (* An exponent state begins with e or E. *)
     exponent,
-    (* Optional plus or minus sign following the exponent. *)
-    exponent_sign,
-    (* Opening a compile-time feature starting with a #. *)
-    number_sign,
     (* Trait identifier. *)
     trait,
-    (* Entering UTF-32 character literal. *)
-    wide,
-    (* Seen an escape sequence in a UTF-32 character literal. *)
-    wide_escape,
-    (* Entering UTF-8 character literal. *)
-    narrow,
-    (* Seen an escape sequence in a UTF-8 character literal. *)
-    narrow_escape,
-    (* Entering string literal. *)
-    string,
-    (* Seen an escape sequence in a string literal. *)
-    string_escape,
     (* Token is complete, the last byte eaten. *)
     stop,
     (* Reject the token with an error. *)
@@ -131,6 +144,103 @@ type
     (* Sentinel signaling end of input. *)
     eof
   )
+  ElnaLexerTokenKind* = (
+    comment,
+    docblock,
+    if_,
+    then_,
+    else_,
+    elsif_,
+    while_,
+    for_,
+    repeat_,
+    until_,
+    with_,
+    do_,
+    proc_,
+    begin_,
+    end_,
+    extern_,
+    const_,
+    var_,
+    threadvar_,
+    type_,
+    record_,
+    true_,
+    false_,
+    and,
+    xor_,
+    or_,
+    pipe,
+    not,
+    return_,
+    import_,
+    cast_,
+    defer_,
+    block_,
+    break_,
+    case_,
+    of_,
+    to_,
+    program_,
+    module_,
+    identifier,
+    trait,
+    generic_open,
+    attribute_open,
+    word,
+    word8,
+    word16,
+    word32,
+    word64,
+    integer,
+    integer8,
+    integer16,
+    integer32,
+    integer64,
+    double,
+    single,
+    character,
+    string,
+    left_paren,
+    right_paren,
+    left_square,
+    right_square,
+    left_brace,
+    right_brace,
+    shift_left,
+    shift_right,
+    greater_equal,
+    less_equal,
+    greater_than,
+    less_than,
+    not_equal,
+    equals,
+    semicolon,
+    dot,
+    comma,
+    plus,
+    minus,
+    multiplication,
+    division,
+    remainder,
+    assignment,
+    colon,
+    hat,
+    at,
+    arrow,
+    eof
+  )
+
+  ElnaLexerCursor* = record
+    state: ElnaLexerState;
+    start, cursor: ^const Word8;
+    position: Word
+  end
+  ElnaLexerToken* = record
+    kind: ElnaLexerTokenKind;
+    position: ElnaSourceSpan
+  end
 
 var
   (* Initializes the array with character classes. *)
@@ -396,7 +506,7 @@ var
     ElnaLexerClass.invalid, -- 0xFE
     ElnaLexerClass.invalid -- 0xFF
   ]
-  transition_table: [cast(#max(ElnaLexerState): Word)][cast(#max(ElnaLexerClass): Word)]ElnaLexerState
+  transition_table: [cast(ElnaLexerState.stop: Word) - 1u][cast(#max(ElnaLexerClass): Word)]ElnaLexerState
 
 (*
   @param current_state First index into transitions table.
@@ -436,12 +546,6 @@ proc elna_lexer_transitions*()
 var
   current := cast(#min(ElnaLexerState): Int)
 begin
-  -- Every transition leads to the dead state by default;
-  -- valid transitions are filled in below.
-  while current <= cast(#max(ElnaLexerState): Int) do
-    elna_lexer_default_transition(cast(current: ElnaLexerState), ElnaLexerState.dead);
-    current := current + 1
-  end;
   -- Start state.
   elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.zero, ElnaLexerState.leading_zero);
   elna_lexer_set_transition(ElnaLexerState.start, ElnaLexerClass.digit, ElnaLexerState.integer);
@@ -585,7 +689,7 @@ begin
   -- Entering ASCII character literal.
   elna_lexer_default_transition(ElnaLexerState.narrow, ElnaLexerState.narrow);
   elna_lexer_set_transition(ElnaLexerState.narrow, ElnaLexerClass.backslash, ElnaLexerState.narrow_escape);
-  elna_lexer_set_transition(ElnaLexerState.narrow, ElnaLexerClass.backquote, ElnaLexerState.stop);
+  elna_lexer_set_transition(ElnaLexerState.narrow, ElnaLexerClass.single_quote, ElnaLexerState.stop);
   elna_lexer_set_transition(ElnaLexerState.narrow, ElnaLexerClass.eof, ElnaLexerState.reject);
   elna_lexer_set_transition(ElnaLexerState.narrow, ElnaLexerClass.invalid, ElnaLexerState.reject);
 
@@ -596,7 +700,7 @@ begin
   -- Entering UTF-8 string literal.
   elna_lexer_default_transition(ElnaLexerState.string, ElnaLexerState.string);
   elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.backslash, ElnaLexerState.string_escape);
-  elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.backquote, ElnaLexerState.stop);
+  elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.double_quote, ElnaLexerState.stop);
   elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.eof, ElnaLexerState.reject);
   elna_lexer_set_transition(ElnaLexerState.string, ElnaLexerClass.invalid, ElnaLexerState.reject);
 
@@ -605,4 +709,119 @@ begin
   elna_lexer_set_transition(ElnaLexerState.string_escape, ElnaLexerClass.invalid, ElnaLexerState.reject)
 return
 
+(*
+  Advances the cursor so that it spans the next lexeme and contains the final
+  accepting state.
+*)
+proc elna_lexer_scan(cursor: ^ElnaLexerCursor)
+var
+  current_state := cursor^.state
+  text_cursor := cursor^.start
+begin
+  repeat
+  var
+    current_class: ElnaLexerClass := classification[cast(text_cursor^ + 1u8: Word)]
+  begin
+    current_state := transition_table[cast(current_state: Word)][cast(current_class: Word)];
+    text_cursor := text_cursor + 1;
+
+    if cast(current_state: Word) >= cast(ElnaLexerState.identifier: Word) then
+      cursor^.state := current_state;
+      cursor^.cursor := text_cursor
+    end
+  until current_state = ElnaLexerState.dead or cast(current_state: Word) >= cast(ElnaLexerState.stop: Word)
+return
+
+(*
+  Prepares the cursor for lexing the next token.
+*)
+proc elna_lexer_reset_cursor(cursor: ^ElnaLexerCursor)
+  cursor^.state := ElnaLexerState.start;
+  cursor^.position := cursor^.position + cast(cursor^.cursor - cursor^.start: Word);
+  cursor^.start := cursor^.cursor
+return
+
+(*
+  Skips blanks and advances the cursor to span the next significant token.
+*)
+proc elna_lexer_skip_trivia(cursor: ^ElnaLexerCursor)
+  repeat
+    elna_lexer_reset_cursor(cursor);
+    elna_lexer_scan(cursor)
+  until cursor^.state <> ElnaLexerState.skip
+return
+
+proc elna_lexer_token_create(kind: ElnaLexerTokenKind) -> ^ElnaLexerToken
+var
+  result: ^ElnaLexerToken
+begin
+  result := malloc#[ElnaLexerToken](#size(ElnaLexerToken));
+  memset(result, 0i8, #size(ElnaLexerToken));
+  result^.kind := kind
+return result
+
+-- TODO: Identifier (else) should save a payload with the identifier.
+proc elna_lexer_identify_keyword(start, cursor: ^const Word8) -> ^ElnaLexerToken
+var
+  result: ^ElnaLexerToken
+  identifier: StdString
+begin
+  identifier := start[1 to cursor - start];
+
+  case identifier of
+    "if": result := elna_lexer_token_create(ElnaLexerTokenKind.if_) |
+    "then": result := elna_lexer_token_create(ElnaLexerTokenKind.then_) |
+    "else": result := elna_lexer_token_create(ElnaLexerTokenKind.else_) |
+    "elsif": result := elna_lexer_token_create(ElnaLexerTokenKind.elsif_) |
+    "while": result := elna_lexer_token_create(ElnaLexerTokenKind.while_) |
+    "for": result := elna_lexer_token_create(ElnaLexerTokenKind.for_) |
+    "repeat": result := elna_lexer_token_create(ElnaLexerTokenKind.repeat_) |
+    "until": result := elna_lexer_token_create(ElnaLexerTokenKind.until_) |
+    "with": result := elna_lexer_token_create(ElnaLexerTokenKind.with_) |
+    "do": result := elna_lexer_token_create(ElnaLexerTokenKind.do_) |
+    "proc": result := elna_lexer_token_create(ElnaLexerTokenKind.proc_) |
+    "begin": result := elna_lexer_token_create(ElnaLexerTokenKind.begin_) |
+    "end": result := elna_lexer_token_create(ElnaLexerTokenKind.end_) |
+    "extern": result := elna_lexer_token_create(ElnaLexerTokenKind.extern_) |
+    "const": result := elna_lexer_token_create(ElnaLexerTokenKind.const_) |
+    "var": result := elna_lexer_token_create(ElnaLexerTokenKind.var_) |
+    "threadvar": result := elna_lexer_token_create(ElnaLexerTokenKind.threadvar_) |
+    "type": result := elna_lexer_token_create(ElnaLexerTokenKind.type_) |
+    "record": result := elna_lexer_token_create(ElnaLexerTokenKind.record_) |
+    "true": result := elna_lexer_token_create(ElnaLexerTokenKind.true_) |
+    "false": result := elna_lexer_token_create(ElnaLexerTokenKind.false_) |
+    "xor": result := elna_lexer_token_create(ElnaLexerTokenKind.xor_) |
+    "or": result := elna_lexer_token_create(ElnaLexerTokenKind.or_) |
+    "return": result := elna_lexer_token_create(ElnaLexerTokenKind.return_) |
+    "import": result := elna_lexer_token_create(ElnaLexerTokenKind.import_) |
+    "cast": result := elna_lexer_token_create(ElnaLexerTokenKind.cast_) |
+    "defer": result := elna_lexer_token_create(ElnaLexerTokenKind.defer_) |
+    "block": result := elna_lexer_token_create(ElnaLexerTokenKind.block_) |
+    "break": result := elna_lexer_token_create(ElnaLexerTokenKind.break_) |
+    "case": result := elna_lexer_token_create(ElnaLexerTokenKind.case_) |
+    "of": result := elna_lexer_token_create(ElnaLexerTokenKind.of_) |
+    "to": result := elna_lexer_token_create(ElnaLexerTokenKind.to_) |
+    "program": result := elna_lexer_token_create(ElnaLexerTokenKind.program_) |
+    "module": result := elna_lexer_token_create(ElnaLexerTokenKind.module_)
+  else
+    result := elna_lexer_token_create(ElnaLexerTokenKind.identifier)
+  end
+return result
+
+-- TODO: Return type should be "error or a pointer".
+proc elna_lexer_lex*(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken
+var
+  result: ^ElnaLexerToken := cast(0: ^ElnaLexerToken)
+begin
+  elna_lexer_skip_trivia(cursor);
+
+  case cursor^.state of
+    ElnaLexerState.identifier: result := elna_lexer_identify_keyword(cursor^.start, cursor^.cursor)
+  else
+    result := malloc#[ElnaLexerToken](#size(ElnaLexerToken));
+    memset(result, 0i8, #size(ElnaLexerToken))
+  end
+
+return result
+
 end.
diff --git a/source/elna/source.elna b/source/elna/source.elna
index 7dfc2ac..af17b45 100644
--- a/source/elna/source.elna
+++ b/source/elna/source.elna
@@ -2,19 +2,13 @@
   v. 2.0. If a copy of the MPL was not distributed with this file, You can
   obtain one at https://mozilla.org/MPL/2.0/. *)
 
-(*
-  Source code representation.
-*)
+(* Source code representation. *)
 module
 
 type
-	ElnaSourceLocation* = record
-		line: Word;
-		column: Word
-	end
-	ElnaSourcePosition* = record
-		start_location: ElnaSourceLocation;
-		end_location: ElnaSourceLocation
-	end
+  ElnaSourceSpan* = record
+    start_offset: Word;
+    end_offset: Word
+  end
 
 end.
diff --git a/source/main.elna b/source/main.elna
index 12486bd..ee4079e 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -2,8 +2,27 @@
   v. 2.0. If a copy of the MPL was not distributed with this file, You can
   obtain one at https://mozilla.org/MPL/2.0/. *)
 program()
-import elna.lexer
+import sys.posix.stdio, elna.lexer
+
+var
+  global_cursor: ElnaLexerCursor
+  sample_source_text := "Here.are a few 'tokens' or if\0"
+  token: ^ElnaLexerToken
 
 begin
-  elna_lexer_transitions()
+  elna_lexer_transitions();
+  global_cursor := { ElnaLexerCursor;
+    start: sample_source_text.ptr,
+    cursor: sample_source_text.ptr,
+    position: 1u
+  };
+
+  repeat
+    token := elna_lexer_lex(@global_cursor);
+
+    printf("Token: %.*s, length: %lu, final state: %i, position: %lu, kind: %i\n\0".ptr,
+        global_cursor.cursor - global_cursor.start, global_cursor.start,
+        global_cursor.cursor - global_cursor.start, global_cursor.state,
+        global_cursor.position, token^.kind)
+  until global_cursor.state = ElnaLexerState.eof
 end.
diff --git a/source/std.elna b/source/std.elna
index 33612dc..31160a3 100644
--- a/source/std.elna
+++ b/source/std.elna
@@ -1,6 +1,9 @@
+(* This Source Code Form is subject to the terms of the Mozilla Public License,
+  v. 2.0. If a copy of the MPL was not distributed with this file, You can
+  obtain one at https://mozilla.org/MPL/2.0/. *)
 module
 
 type
-  String* = []Word8
+  StdString* = []const Word8
 
 end.
diff --git a/source/sys/posix/stdio.elna b/source/sys/posix/stdio.elna
new file mode 100644
index 0000000..dd7251d
--- /dev/null
+++ b/source/sys/posix/stdio.elna
@@ -0,0 +1,9 @@
+(* This Source Code Form is subject to the terms of the Mozilla Public License,
+  v. 2.0. If a copy of the MPL was not distributed with this file, You can
+  obtain one at https://mozilla.org/MPL/2.0/. *)
+module
+
+proc printf*(format: ^const Word8) #{varargs} -> Int
+extern
+
+end.
diff --git a/source/sys/posix/stdlib.elna b/source/sys/posix/stdlib.elna
new file mode 100644
index 0000000..0016d9c
--- /dev/null
+++ b/source/sys/posix/stdlib.elna
@@ -0,0 +1,9 @@
+(* This Source Code Form is subject to the terms of the Mozilla Public License,
+  v. 2.0. If a copy of the MPL was not distributed with this file, You can
+  obtain one at https://mozilla.org/MPL/2.0/. *)
+module
+
+proc malloc#[T]*(size: Word) -> ^T
+extern
+
+end.
diff --git a/source/sys/posix/string.elna b/source/sys/posix/string.elna
new file mode 100644
index 0000000..9994679
--- /dev/null
+++ b/source/sys/posix/string.elna
@@ -0,0 +1,9 @@
+(* This Source Code Form is subject to the terms of the Mozilla Public License,
+  v. 2.0. If a copy of the MPL was not distributed with this file, You can
+  obtain one at https://mozilla.org/MPL/2.0/. *)
+module
+
+proc memset#[T]*(b: ^T; c: Int8; len: Word) -> ^T
+extern
+
+end.