Fix error span for missing return message

6 files changed, 154 insertions, 42 deletions

diff --git a/boot/name_analysis.cc b/boot/name_analysis.cc
index 11380db..9305cd2 100644
--- a/boot/name_analysis.cc
+++ b/boot/name_analysis.cc
@@ -1628,8 +1628,7 @@ namespace elna::boot
         this->bag.leave();
         info_object->parameters = type_parameters;
         info_object->exported = declaration->identifier.exported();
-        info_object->position.emplace(declaration->position().start(),
-                declaration->heading().position().end());
+        info_object->position.emplace(declaration->position());
         info_object->file = this->module_file;
         info_object->noreturn = noreturn;
 
diff --git a/boot/parser.yy b/boot/parser.yy
index fa701ec..c4e514b 100644
--- a/boot/parser.yy
+++ b/boot/parser.yy
@@ -328,12 +328,15 @@ procedure_heading:
 procedure_declaration:
     "proc" identifier_definition procedure_heading procedure_body
         {
-            $$ = new boot::procedure_declaration(boot::make_position(@$),
+            $$ = new boot::procedure_declaration(
+                    boot::make_position(yy::location(@1.begin, @3.end)),
                     std::move(*$2), $3, std::move(*$4));
         }
     | "proc" identifier_definition procedure_heading "extern"
         {
-            $$ = new boot::procedure_declaration(boot::make_position(@$), std::move(*$2), $3);
+            $$ = new boot::procedure_declaration(
+                    boot::make_position(yy::location(@1.begin, @3.end)),
+                    std::move(*$2), $3);
         }
 call_expression:
     designator_expression actual_parameter_list
diff --git a/boot/type_check.cc b/boot/type_check.cc
index 14a5099..7adaaed 100644
--- a/boot/type_check.cc
+++ b/boot/type_check.cc
@@ -198,7 +198,7 @@ namespace elna::boot
                         + this->actual.to_string() + "'";
                 }
                 return describe_callable(payload.identifier)
-                    + " is expected to return, but does not have a return statement";
+                    + " is expected to return a value, but has no return expression";
 
             }
             else if constexpr (std::is_same_v<T, unary>)
diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna
index cef6687..a543c67 100644
--- a/source/elna/lexer.elna
+++ b/source/elna/lexer.elna
@@ -252,6 +252,10 @@ type
     eof
   )
 
+  ElnaLexerConsumedCodepoint = record
+    consumed: Int;
+    codepoint: Word32
+  end
   ElnaLexerCursor* = record
     state: ElnaLexerState;
     start, cursor: ^const Word8;
@@ -270,6 +274,9 @@ type
   ElnaLexerWordToken = record(ElnaLexerToken)
     value: Word64
   end
+  ElnaLexerCharacterToken = record(ElnaLexerToken)
+    codepoint: Word32
+  end
 
 const
   ELNA_LEXER_UTF8_LEAD_TAG_2BYTE = 0xC0u8
@@ -845,38 +852,82 @@ begin
   end
 return encoded
 
-proc elna_lexer_parse_codepoint_notation(current_position: ^const Word8; end_pointer: ^^const Word8) -> Word32
+proc elna_lexer_decode_utf8(current_position: ^const Word8) -> ElnaLexerConsumedCodepoint
+const
+  LEAD_TAG_4BYTE_TEST = 0xF8u8
+  LEAD_PAYLOAD_MASK_2BYTE = 0x1Fu32
+  LEAD_PAYLOAD_MASK_3BYTE = 0x0Fu32
+  LEAD_PAYLOAD_MASK_4BYTE = 0x07u32
+  ASCII_BIT_MASK = 0x80u8
+var
+  lead: const := current_position^
+  extra, i: Int
+  codepoint := #max(Word32)
+begin
+  if (lead & ASCII_BIT_MASK) = 0x00u8 then
+      extra := 0;
+      codepoint := cast(lead: Word32)
+  elsif (lead & ELNA_LEXER_UTF8_LEAD_TAG_3BYTE) = ELNA_LEXER_UTF8_LEAD_TAG_2BYTE then
+    extra := 1;
+    codepoint := cast(lead: Word32) & LEAD_PAYLOAD_MASK_2BYTE;
+  elsif (lead & ELNA_LEXER_UTF8_LEAD_TAG_4BYTE) = ELNA_LEXER_UTF8_LEAD_TAG_3BYTE then
+    extra := 2;
+    codepoint := cast(lead: Word32) & LEAD_PAYLOAD_MASK_3BYTE
+  elsif (lead & LEAD_TAG_4BYTE_TEST) = ELNA_LEXER_UTF8_LEAD_TAG_4BYTE then
+    extra := 3;
+    codepoint := cast(lead: Word32) & LEAD_PAYLOAD_MASK_4BYTE
+  else
+    extra := -1
+  end;
+  i := 1;
+  while i <= extra do
+  var
+    continuation_byte: const := (current_position + i)^
+  begin
+    if (continuation_byte & ELNA_LEXER_UTF8_LEAD_TAG_2BYTE) = ELNA_LEXER_UTF8_CONTINUATION_TAG then
+      codepoint := (codepoint << ELNA_LEXER_UTF8_CONTINUATION_BITS)
+        or (cast(continuation_byte: Word32) & ELNA_LEXER_UTF8_CONTINUATION_MASK);
+      i := i + 1
+    else
+      extra := -1
+    end
+  end
+return { ElnaLexerConsumedCodepoint; codepoint: codepoint, consumed: extra + 1 }
+
+proc elna_lexer_parse_codepoint_notation(current_position: ^const Word8) -> ElnaLexerConsumedCodepoint
 var
   codepoint := #max(Word32) -- Signal an error with an invalid unicode value.
+  consumed := 0
 begin
-  if current_position^ = 'U' or (current_position + 1)^ = '+' then
+  if current_position^ = 'U' & (current_position + 1)^ = '+' then
   var
-    consumed := 0
+    digits_start: ^const Word8
   begin
     current_position := current_position + 2;
-    end_pointer^ := current_position;
+    digits_start := current_position;
 
     while elna_lexer_is_hex(current_position^) do
       current_position := current_position + 1
     end;
-    consumed := current_position - end_pointer^;
+    consumed := 2 + (current_position - digits_start);
 
-    if consumed >= 0 & consumed <= 6 then -- Unicode maximum is U+10FFFF.
+    if current_position <> digits_start & consumed <= 8 then -- Unicode maximum is U+10FFFF.
     var
       hex_digits := [0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]
     begin
-      memcpy(hex_digits.ptr, current_position, cast(consumed: Word));
+      memcpy(hex_digits.ptr, digits_start, cast(current_position - digits_start: Word));
       codepoint := cast(strtoul(hex_digits.ptr, cast(0: ^^Word8), 16): Word32);
 
       if (codepoint >= ELNA_LEXER_UTF8_SURROGATE_RANGE_START & codepoint <= ELNA_LEXER_UTF8_SURROGATE_RANGE_END)
           or codepoint > ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT then
         codepoint := #max(Word32)
-      else
-        end_pointer^ := current_position
       end
     end
+  end;
+  if codepoint = #max(Word32) then
+    consumed := 0
   end
-return codepoint
+return { ElnaLexerConsumedCodepoint; codepoint: codepoint, consumed: consumed }
 
 proc elna_lexer_decode_one_character(current_position: ^const Word8; output: ^^Word8) -> Int
 var
@@ -896,32 +947,33 @@ begin
     end
   elsif current_position^ = '\\' & (current_position + 1)^ = '{' then
   var
-    codepoint := #max(Word32)
-    encoded: Int
-    buffer: [4]Word8
-    end_pointer := current_position
+    unescaped: ElnaLexerConsumedCodepoint
   begin
     current_position := current_position + 2;
-    codepoint := elna_lexer_parse_codepoint_notation(current_position, @end_pointer);
+    unescaped := elna_lexer_parse_codepoint_notation(current_position);
+    current_position := current_position + unescaped.consumed;
 
-    if codepoint <> #max(Word32) & current_position^ = '}' then
-      encoded := elna_lexer_encode_utf8(codepoint, @buffer);
+    if unescaped.consumed <> 0 & current_position^ = '}' then
+    var
+      encoded: Int
+      buffer: [4]Word8
+    begin
+      consumed := unescaped.consumed;
+      encoded := elna_lexer_encode_utf8(unescaped.codepoint, @buffer);
 
       while encoded > 0 do
         output^^ := buffer[encoded];
         output^ := output^ + 1;
         encoded := encoded - 1
       end;
-      consumed := 3 + (end_pointer - current_position)
+      consumed := consumed + 3
     end
   elsif current_position^ = '\\' then
   var
-    escape := ELNA_LEXER_ESCAPE_INVALID_CHAR
+    escape := elna_lexer_unescape_character((current_position + 1)^)
   begin
-    current_position := current_position + 1;
-    escape := elna_lexer_unescape_character(current_position^);
-
     if escape <> ELNA_LEXER_ESCAPE_INVALID_CHAR then
+      current_position := current_position + 2;
       output^^ := escape;
       consumed := 2
     end
@@ -931,11 +983,57 @@ begin
   end
 return consumed
 
-proc elna_lexer_parse_byte_literal(escape: ^const Word8) -> StdOptional#[ElnaLexerToken]
+proc elna_lexer_parse_character_literal(character_literal: ^const Word8) -> StdOptional#[ElnaLexerToken]
+var
+  current_position := character_literal + 1 
+  codepoint := #max(Word32)
+  result := { StdOptional#[ElnaLexerToken] }
+begin
+  if current_position^ = '\\' & (current_position + 1)^ = '{' then
+  var
+    unescaped: ElnaLexerConsumedCodepoint
+  begin
+    current_position := current_position + 2;
+    unescaped := elna_lexer_parse_codepoint_notation(current_position);
+    current_position := current_position + unescaped.consumed;
+
+    if unescaped.consumed <> 0 & current_position^ = '}' then 
+      codepoint := unescaped.codepoint;
+      current_position := current_position + 1
+    end
+  elsif current_position^ = '\\' then
+  var
+    escape := elna_lexer_unescape_character((current_position + 1)^)
+  begin
+    if escape <> ELNA_LEXER_ESCAPE_INVALID_CHAR then
+      current_position := current_position + 2;
+      codepoint := cast(escape: Word32)
+    end
+  else
+  var
+    decoded := elna_lexer_decode_utf8(current_position)
+  begin
+    if decoded.consumed <> 0 then
+      codepoint := decoded.codepoint;
+      current_position := current_position + decoded.consumed;
+    end
+  end;
+  if codepoint <> #max(Word32) & current_position^ = '`' then
+  var
+    token := malloc#[ElnaLexerCharacterToken](#size(ElnaLexerCharacterToken))
+  begin
+    token^.kind := ElnaLexerTokenKind.character;
+    token^.codepoint := codepoint;
+
+    result := std_optional_make#[ElnaLexerToken](token)
+  end
+return result
+
+proc elna_lexer_parse_byte_literal(byte_literal: ^const Word8) -> StdOptional#[ElnaLexerToken]
 var
   buffer: [4]Word8
   output := buffer.ptr
-  current_position := escape + 1
+  current_position := byte_literal + 1
   consumed := elna_lexer_decode_one_character(current_position, @output)
   result := { StdOptional#[ElnaLexerToken] }
 begin
@@ -1100,6 +1198,8 @@ begin
     result := std_optional_make#[ElnaLexerToken](comment_token)
   elsif start^ = '\'' then
     result := elna_lexer_parse_byte_literal(start)
+  elsif start^ = '`' then
+    result := elna_lexer_parse_character_literal(start)
   end
 return result
 
@@ -1207,15 +1307,15 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
     ElnaLexerTokenKind.generic_open: printf("«#[»\0".ptr) |
     ElnaLexerTokenKind.attribute_open: printf("«#{»\0".ptr) |
     ElnaLexerTokenKind.word:
-      printf("«0x%xu»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+      printf("«0x%Xu»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
     ElnaLexerTokenKind.word8:
-      printf("«0x%02xu8»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+      printf("«0x%02Xu8»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
     ElnaLexerTokenKind.word16:
-      printf("«0x%04xu16»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+      printf("«0x%04Xu16»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
     ElnaLexerTokenKind.word32:
-      printf("«0x%08xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+      printf("«0x%08Xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
     ElnaLexerTokenKind.word64:
-      printf("«0x%016lxu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+      printf("«0x%016lXu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
     ElnaLexerTokenKind.integer: printf("integer\0".ptr) |
     ElnaLexerTokenKind.integer8: printf("integer8\0".ptr) |
     ElnaLexerTokenKind.integer16: printf("integer16\0".ptr) |
@@ -1223,7 +1323,8 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
     ElnaLexerTokenKind.integer64: printf("integer64\0".ptr) |
     ElnaLexerTokenKind.double: printf("double\0".ptr) |
     ElnaLexerTokenKind.single: printf("single\0".ptr) |
-    ElnaLexerTokenKind.character: printf("character\0".ptr) |
+    ElnaLexerTokenKind.character:
+      printf("\\{U+%X}\0".ptr, cast(token: ^ElnaLexerCharacterToken)^.codepoint) |
     ElnaLexerTokenKind.string: printf("string\0".ptr) |
     ElnaLexerTokenKind.left_paren: printf("«(»\0".ptr) |
     ElnaLexerTokenKind.right_paren: printf("«)»\0".ptr) |
diff --git a/source/main.elna b/source/main.elna
index acc5674..7065b3b 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer
 
 var
   global_cursor: ElnaLexerCursor
-  sample_source_text := "Here.are a few (* documentation block *) or if '\\n' -- End line comment\0"
+  sample_source_text := "Here.are a few (* documentation block *) or if '\\n' `У` -- End line comment\0"
   token: ^ElnaLexerToken
 
 begin
diff --git a/tools/tree-sitter/grammar.js b/tools/tree-sitter/grammar.js
index 4aab2fa..5d2dda7 100644
--- a/tools/tree-sitter/grammar.js
+++ b/tools/tree-sitter/grammar.js
@@ -150,12 +150,21 @@ module.exports = grammar({
 
     variable_declaration: $ => seq(
       field('names', commaSep1($.identifier_definition)),
-      ':',
-      field('type', $._type_expression),
-      optional(seq(
-        ':=',
-        field('value', choice($._expression, 'extern')),
-      )),
+      choice(
+        seq(
+          ':',
+          field('type', $._type_expression),
+          optional(seq(
+            ':=',
+            field('value', choice($._expression, 'extern')),
+          )),
+        ),
+        seq(
+          optional(seq(':', 'const')),
+          ':=',
+          field('value', $._expression),
+        ),
+      ),
     ),
 
     procedure_declaration: $ => seq(