Fix type checker traversal for record and array literals

4 files changed, 187 insertions, 14 deletions

diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna
index 1130b7a..cef6687 100644
--- a/source/elna/lexer.elna
+++ b/source/elna/lexer.elna
@@ -267,6 +267,9 @@ type
   ElnaLexerCommentToken = record(ElnaLexerToken)
     content: StdString
   end
+  ElnaLexerWordToken = record(ElnaLexerToken)
+    value: Word64
+  end
 
 const
   ELNA_LEXER_UTF8_LEAD_TAG_2BYTE = 0xC0u8
@@ -281,6 +284,8 @@ const
   ELNA_LEXER_UTF8_SURROGATE_RANGE_END = 0xDFFFu32
   ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT = 0x10FFFFu32
 
+  ELNA_LEXER_ESCAPE_INVALID_CHAR = '\xff'
+
 var
   (* Initializes the array with character classes. *)
   classification: const [256]ElnaLexerClass := [
@@ -780,44 +785,174 @@ proc elna_lexer_reset_cursor(cursor: ^ElnaLexerCursor)
   cursor^.start := cursor^.cursor
 return
 
+proc elna_lexer_unescape_character(escape: Word8) -> Word8
+var
+  escaped: Word8
+begin
+    case escape of
+      'n': escaped := '\n' |
+      't': escaped := '\t' |
+      'f': escaped := '\f' |
+      'r': escaped := '\r' |
+      'v': escaped := '\v' |
+      '\\': escaped := '\\' |
+      '\'': escaped := '\'' |
+      '"': escaped := '"' |
+      '`': escaped := '`' |
+      '0': escaped := '\0'
+    else
+      escaped := ELNA_LEXER_ESCAPE_INVALID_CHAR
+    end
+return escaped
+
 proc elna_lexer_is_hex(character: Word8) -> Bool
 return (character >= '0' & character <= '9')
   or (character >= 'a' & character <= 'f')
   or (character >= 'A' & character <= 'F')
 
-proc encode_utf8(codepoint: Word32; buffer: ^[4]Word8) -> Word
+proc elna_lexer_encode_utf8(codepoint: Word32; buffer: ^[4]Word8) -> Int
 const
   UTF8_1BYTE_MAX = 0x80u32
   UTF8_2BYTE_MAX = 0x800u32
   UTF8_3BYTE_MAX = 0x10000u32
 var
-  encoded: Word
+  encoded: Int
 begin
   if (codepoint >= ELNA_LEXER_UTF8_SURROGATE_RANGE_START & codepoint <= ELNA_LEXER_UTF8_SURROGATE_RANGE_END)
       or codepoint > ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT then
-    encoded := 0u
+    encoded := 0
   elsif codepoint < UTF8_1BYTE_MAX then
     buffer^[1] := cast(codepoint: Word8);
-    encoded := 1u
+    encoded := 1
   elsif codepoint < UTF8_2BYTE_MAX then
     buffer^[1] := ELNA_LEXER_UTF8_CONTINUATION_TAG or cast(codepoint & ELNA_LEXER_UTF8_CONTINUATION_MASK: Word8);
     buffer^[2] := ELNA_LEXER_UTF8_LEAD_TAG_2BYTE or cast(codepoint >> ELNA_LEXER_UTF8_CONTINUATION_BITS: Word8);
-    encoded := 2u
+    encoded := 2
   elsif codepoint < UTF8_3BYTE_MAX then
     for i of buffer^[1 to 2] do
       i^ := ELNA_LEXER_UTF8_CONTINUATION_TAG or cast(codepoint & ELNA_LEXER_UTF8_CONTINUATION_MASK: Word8);
       codepoint := codepoint >> ELNA_LEXER_UTF8_CONTINUATION_BITS
     end;
-    buffer^[3] := ELNA_LEXER_UTF8_LEAD_TAG_3BYTE or cast(codepoint: Word8)
+    buffer^[3] := ELNA_LEXER_UTF8_LEAD_TAG_3BYTE or cast(codepoint: Word8);
+    encoded := 3
   else
     for i of buffer^[1 to 3] do
       i^ := ELNA_LEXER_UTF8_CONTINUATION_TAG or cast(codepoint & ELNA_LEXER_UTF8_CONTINUATION_MASK: Word8);
       codepoint := codepoint >> ELNA_LEXER_UTF8_CONTINUATION_BITS
     end;
-    buffer^[4] := ELNA_LEXER_UTF8_LEAD_TAG_4BYTE or cast(codepoint: Word8)
+    buffer^[4] := ELNA_LEXER_UTF8_LEAD_TAG_4BYTE or cast(codepoint: Word8);
+    encoded := 4
   end
 return encoded
 
+proc elna_lexer_parse_codepoint_notation(current_position: ^const Word8; end_pointer: ^^const Word8) -> Word32
+var
+  codepoint := #max(Word32) -- Signal an error with an invalid unicode value.
+begin
+  if current_position^ = 'U' or (current_position + 1)^ = '+' then
+  var
+    consumed := 0
+  begin
+    current_position := current_position + 2;
+    end_pointer^ := current_position;
+
+    while elna_lexer_is_hex(current_position^) do
+      current_position := current_position + 1
+    end;
+    consumed := current_position - end_pointer^;
+
+    if consumed >= 0 & consumed <= 6 then -- Unicode maximum is U+10FFFF.
+    var
+      hex_digits := [0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]
+    begin
+      memcpy(hex_digits.ptr, current_position, cast(consumed: Word));
+      codepoint := cast(strtoul(hex_digits.ptr, cast(0: ^^Word8), 16): Word32);
+
+      if (codepoint >= ELNA_LEXER_UTF8_SURROGATE_RANGE_START & codepoint <= ELNA_LEXER_UTF8_SURROGATE_RANGE_END)
+          or codepoint > ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT then
+        codepoint := #max(Word32)
+      else
+        end_pointer^ := current_position
+      end
+    end
+  end
+return codepoint
+
+proc elna_lexer_decode_one_character(current_position: ^const Word8; output: ^^Word8) -> Int
+var
+  consumed := 0
+begin
+  if current_position^ = '\\' & (current_position + 1)^ = 'x' then
+    current_position := current_position + 2;
+
+    if elna_lexer_is_hex(current_position^) & elna_lexer_is_hex((current_position + 1)^) then
+    var
+      hex_digits: const := [current_position^, (current_position + 1)^, 0u8]
+      character: const := cast(strtol(hex_digits.ptr, cast(0: ^^Word8), 16): Word8)
+    begin
+      output^^ := character;
+      output^ := output^ + 1;
+      consumed := 4
+    end
+  elsif current_position^ = '\\' & (current_position + 1)^ = '{' then
+  var
+    codepoint := #max(Word32)
+    encoded: Int
+    buffer: [4]Word8
+    end_pointer := current_position
+  begin
+    current_position := current_position + 2;
+    codepoint := elna_lexer_parse_codepoint_notation(current_position, @end_pointer);
+
+    if codepoint <> #max(Word32) & current_position^ = '}' then
+      encoded := elna_lexer_encode_utf8(codepoint, @buffer);
+
+      while encoded > 0 do
+        output^^ := buffer[encoded];
+        output^ := output^ + 1;
+        encoded := encoded - 1
+      end;
+      consumed := 3 + (end_pointer - current_position)
+    end
+  elsif current_position^ = '\\' then
+  var
+    escape := ELNA_LEXER_ESCAPE_INVALID_CHAR
+  begin
+    current_position := current_position + 1;
+    escape := elna_lexer_unescape_character(current_position^);
+
+    if escape <> ELNA_LEXER_ESCAPE_INVALID_CHAR then
+      output^^ := escape;
+      consumed := 2
+    end
+  else
+    output^^ := current_position^;
+    consumed := 1
+  end
+return consumed
+
+proc elna_lexer_parse_byte_literal(escape: ^const Word8) -> StdOptional#[ElnaLexerToken]
+var
+  buffer: [4]Word8
+  output := buffer.ptr
+  current_position := escape + 1
+  consumed := elna_lexer_decode_one_character(current_position, @output)
+  result := { StdOptional#[ElnaLexerToken] }
+begin
+  current_position := current_position + consumed;
+
+  -- Unicode consumes more than 4 characters and is not supported here.
+  if consumed > 0 & consumed <= 4 & current_position^ = '\'' then
+  var
+    byte_token := malloc#[ElnaLexerWordToken](#size(ElnaLexerWordToken))
+  begin
+    byte_token^.kind := ElnaLexerTokenKind.word8;
+    byte_token^.value := cast(buffer[1]: Word64);
+
+    result := std_optional_make#[ElnaLexerToken](byte_token)
+  end
+return result
+
 (*
   Skips blanks and advances the cursor to span the next significant token.
 *)
@@ -954,6 +1089,20 @@ begin
   comment_token^.content := comment_pointer[1 to comment_length]
 return comment_token
 
+proc elna_lexer_delimit_string(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken]
+var
+  result := { StdOptional#[ElnaLexerToken] }
+begin
+  if start^ = '(' then
+  var
+    comment_token := elna_lexer_stop_comment(ElnaLexerTokenKind.docblock, start, cursor - 2u)
+  begin
+    result := std_optional_make#[ElnaLexerToken](comment_token)
+  elsif start^ = '\'' then
+    result := elna_lexer_parse_byte_literal(start)
+  end
+return result
+
 -- TODO: Return type should be "error or a pointer".
 proc elna_lexer_lex*(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken
 var
@@ -963,7 +1112,7 @@ begin
 
   case cursor^.state of
     ElnaLexerState.stop:
-      result := elna_lexer_stop_comment(ElnaLexerTokenKind.docblock, cursor^.start, cursor^.cursor - 2u)
+      result := elna_lexer_delimit_string(cursor^.start, cursor^.cursor - 2u).value
     |
     ElnaLexerState.comment:
       result := elna_lexer_stop_comment(ElnaLexerTokenKind.comment, cursor^.start, cursor^.cursor)
@@ -1057,11 +1206,16 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
     ElnaLexerTokenKind.trait: printf("trait\0".ptr) |
     ElnaLexerTokenKind.generic_open: printf("«#[»\0".ptr) |
     ElnaLexerTokenKind.attribute_open: printf("«#{»\0".ptr) |
-    ElnaLexerTokenKind.word: printf("word\0".ptr) |
-    ElnaLexerTokenKind.word8: printf("word8\0".ptr) |
-    ElnaLexerTokenKind.word16: printf("word16\0".ptr) |
-    ElnaLexerTokenKind.word32: printf("word32\0".ptr) |
-    ElnaLexerTokenKind.word64: printf("word64\0".ptr) |
+    ElnaLexerTokenKind.word:
+      printf("«0x%xu»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+    ElnaLexerTokenKind.word8:
+      printf("«0x%02xu8»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+    ElnaLexerTokenKind.word16:
+      printf("«0x%04xu16»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+    ElnaLexerTokenKind.word32:
+      printf("«0x%08xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+    ElnaLexerTokenKind.word64:
+      printf("«0x%016lxu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
     ElnaLexerTokenKind.integer: printf("integer\0".ptr) |
     ElnaLexerTokenKind.integer8: printf("integer8\0".ptr) |
     ElnaLexerTokenKind.integer16: printf("integer16\0".ptr) |
diff --git a/source/main.elna b/source/main.elna
index 2099cb3..acc5674 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer
 
 var
   global_cursor: ElnaLexerCursor
-  sample_source_text := "Here.are a few (* documentation block *) or if -- End line comment\0"
+  sample_source_text := "Here.are a few (* documentation block *) or if '\\n' -- End line comment\0"
   token: ^ElnaLexerToken
 
 begin
diff --git a/source/std.elna b/source/std.elna
index 31160a3..c028b93 100644
--- a/source/std.elna
+++ b/source/std.elna
@@ -5,5 +5,18 @@ module
 
 type
   StdString* = []const Word8
+  StdOptional#[T]* = record
+    has_value: Bool;
+    value: ^T
+  end
+
+(*
+  Constructs an optional holding the pointer value.
+
+  @param value Initial optional value.
+  @return Constructed optional.
+*)
+proc std_optional_make#[T]*(value: ^T) -> StdOptional#[T]
+return { StdOptional#[T]; has_value: true, value: value }
 
 end.
diff --git a/source/sys/posix/stdlib.elna b/source/sys/posix/stdlib.elna
index 66b383a..8746970 100644
--- a/source/sys/posix/stdlib.elna
+++ b/source/sys/posix/stdlib.elna
@@ -9,4 +9,10 @@ extern
 proc free#[T]*(ptr: ^T)
 extern
 
+proc strtol*(str: ^const Word8; endptr: ^^Word8; base: Int) -> Int
+extern
+
+proc strtoul*(str: ^const Word8; endptr: ^^Word8; base: Int) -> Word
+extern
+
 end.