aboutsummaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
Diffstat (limited to 'source')
-rw-r--r--source/elna/lexer.elna161
-rw-r--r--source/main.elna2
2 files changed, 132 insertions, 31 deletions
diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna
index cef6687..a543c67 100644
--- a/source/elna/lexer.elna
+++ b/source/elna/lexer.elna
@@ -252,6 +252,10 @@ type
eof
)
+ ElnaLexerConsumedCodepoint = record
+ consumed: Int;
+ codepoint: Word32
+ end
ElnaLexerCursor* = record
state: ElnaLexerState;
start, cursor: ^const Word8;
@@ -270,6 +274,9 @@ type
ElnaLexerWordToken = record(ElnaLexerToken)
value: Word64
end
+ ElnaLexerCharacterToken = record(ElnaLexerToken)
+ codepoint: Word32
+ end
const
ELNA_LEXER_UTF8_LEAD_TAG_2BYTE = 0xC0u8
@@ -845,38 +852,82 @@ begin
end
return encoded
-proc elna_lexer_parse_codepoint_notation(current_position: ^const Word8; end_pointer: ^^const Word8) -> Word32
+proc elna_lexer_decode_utf8(current_position: ^const Word8) -> ElnaLexerConsumedCodepoint
+const
+ LEAD_TAG_4BYTE_TEST = 0xF8u8
+ LEAD_PAYLOAD_MASK_2BYTE = 0x1Fu32
+ LEAD_PAYLOAD_MASK_3BYTE = 0x0Fu32
+ LEAD_PAYLOAD_MASK_4BYTE = 0x07u32
+ ASCII_BIT_MASK = 0x80u8
+var
+ lead: const := current_position^
+ extra, i: Int
+ codepoint := #max(Word32)
+begin
+ if (lead & ASCII_BIT_MASK) = 0x00u8 then
+ extra := 0;
+ codepoint := cast(lead: Word32)
+ elsif (lead & ELNA_LEXER_UTF8_LEAD_TAG_3BYTE) = ELNA_LEXER_UTF8_LEAD_TAG_2BYTE then
+ extra := 1;
+ codepoint := cast(lead: Word32) & LEAD_PAYLOAD_MASK_2BYTE;
+ elsif (lead & ELNA_LEXER_UTF8_LEAD_TAG_4BYTE) = ELNA_LEXER_UTF8_LEAD_TAG_3BYTE then
+ extra := 2;
+ codepoint := cast(lead: Word32) & LEAD_PAYLOAD_MASK_3BYTE
+ elsif (lead & LEAD_TAG_4BYTE_TEST) = ELNA_LEXER_UTF8_LEAD_TAG_4BYTE then
+ extra := 3;
+ codepoint := cast(lead: Word32) & LEAD_PAYLOAD_MASK_4BYTE
+ else
+ extra := -1
+ end;
+ i := 1;
+ while i <= extra do
+ var
+ continuation_byte: const := (current_position + i)^
+ begin
+ if (continuation_byte & ELNA_LEXER_UTF8_LEAD_TAG_2BYTE) = ELNA_LEXER_UTF8_CONTINUATION_TAG then
+ codepoint := (codepoint << ELNA_LEXER_UTF8_CONTINUATION_BITS)
+ or (cast(continuation_byte: Word32) & ELNA_LEXER_UTF8_CONTINUATION_MASK);
+ i := i + 1
+ else
+ extra := -1
+ end
+ end
+return { ElnaLexerConsumedCodepoint; codepoint: codepoint, consumed: extra + 1 }
+
+proc elna_lexer_parse_codepoint_notation(current_position: ^const Word8) -> ElnaLexerConsumedCodepoint
var
codepoint := #max(Word32) -- Signal an error with an invalid unicode value.
+ consumed := 0
begin
- if current_position^ = 'U' or (current_position + 1)^ = '+' then
+ if current_position^ = 'U' & (current_position + 1)^ = '+' then
var
- consumed := 0
+ digits_start: ^const Word8
begin
current_position := current_position + 2;
- end_pointer^ := current_position;
+ digits_start := current_position;
while elna_lexer_is_hex(current_position^) do
current_position := current_position + 1
end;
- consumed := current_position - end_pointer^;
+ consumed := 2 + (current_position - digits_start);
- if consumed >= 0 & consumed <= 6 then -- Unicode maximum is U+10FFFF.
+ if current_position <> digits_start & consumed <= 8 then -- Unicode maximum is U+10FFFF.
var
hex_digits := [0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]
begin
- memcpy(hex_digits.ptr, current_position, cast(consumed: Word));
+ memcpy(hex_digits.ptr, digits_start, cast(current_position - digits_start: Word));
codepoint := cast(strtoul(hex_digits.ptr, cast(0: ^^Word8), 16): Word32);
if (codepoint >= ELNA_LEXER_UTF8_SURROGATE_RANGE_START & codepoint <= ELNA_LEXER_UTF8_SURROGATE_RANGE_END)
or codepoint > ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT then
codepoint := #max(Word32)
- else
- end_pointer^ := current_position
end
end
+ end;
+ if codepoint = #max(Word32) then
+ consumed := 0
end
-return codepoint
+return { ElnaLexerConsumedCodepoint; codepoint: codepoint, consumed: consumed }
proc elna_lexer_decode_one_character(current_position: ^const Word8; output: ^^Word8) -> Int
var
@@ -896,32 +947,33 @@ begin
end
elsif current_position^ = '\\' & (current_position + 1)^ = '{' then
var
- codepoint := #max(Word32)
- encoded: Int
- buffer: [4]Word8
- end_pointer := current_position
+ unescaped: ElnaLexerConsumedCodepoint
begin
current_position := current_position + 2;
- codepoint := elna_lexer_parse_codepoint_notation(current_position, @end_pointer);
+ unescaped := elna_lexer_parse_codepoint_notation(current_position);
+ current_position := current_position + unescaped.consumed;
- if codepoint <> #max(Word32) & current_position^ = '}' then
- encoded := elna_lexer_encode_utf8(codepoint, @buffer);
+ if unescaped.consumed <> 0 & current_position^ = '}' then
+ var
+ encoded: Int
+ buffer: [4]Word8
+ begin
+ consumed := unescaped.consumed;
+ encoded := elna_lexer_encode_utf8(unescaped.codepoint, @buffer);
while encoded > 0 do
output^^ := buffer[encoded];
output^ := output^ + 1;
encoded := encoded - 1
end;
- consumed := 3 + (end_pointer - current_position)
+ consumed := consumed + 3
end
elsif current_position^ = '\\' then
var
- escape := ELNA_LEXER_ESCAPE_INVALID_CHAR
+ escape := elna_lexer_unescape_character((current_position + 1)^)
begin
- current_position := current_position + 1;
- escape := elna_lexer_unescape_character(current_position^);
-
if escape <> ELNA_LEXER_ESCAPE_INVALID_CHAR then
+ current_position := current_position + 2;
output^^ := escape;
consumed := 2
end
@@ -931,11 +983,57 @@ begin
end
return consumed
-proc elna_lexer_parse_byte_literal(escape: ^const Word8) -> StdOptional#[ElnaLexerToken]
+proc elna_lexer_parse_character_literal(character_literal: ^const Word8) -> StdOptional#[ElnaLexerToken]
+var
+ current_position := character_literal + 1
+ codepoint := #max(Word32)
+ result := { StdOptional#[ElnaLexerToken] }
+begin
+ if current_position^ = '\\' & (current_position + 1)^ = '{' then
+ var
+ unescaped: ElnaLexerConsumedCodepoint
+ begin
+ current_position := current_position + 2;
+ unescaped := elna_lexer_parse_codepoint_notation(current_position);
+ current_position := current_position + unescaped.consumed;
+
+ if unescaped.consumed <> 0 & current_position^ = '}' then
+ codepoint := unescaped.codepoint;
+ current_position := current_position + 1
+ end
+ elsif current_position^ = '\\' then
+ var
+ escape := elna_lexer_unescape_character((current_position + 1)^)
+ begin
+ if escape <> ELNA_LEXER_ESCAPE_INVALID_CHAR then
+ current_position := current_position + 2;
+ codepoint := cast(escape: Word32)
+ end
+ else
+ var
+ decoded := elna_lexer_decode_utf8(current_position)
+ begin
+ if decoded.consumed <> 0 then
+ codepoint := decoded.codepoint;
+ current_position := current_position + decoded.consumed;
+ end
+ end;
+ if codepoint <> #max(Word32) & current_position^ = '`' then
+ var
+ token := malloc#[ElnaLexerCharacterToken](#size(ElnaLexerCharacterToken))
+ begin
+ token^.kind := ElnaLexerTokenKind.character;
+ token^.codepoint := codepoint;
+
+ result := std_optional_make#[ElnaLexerToken](token)
+ end
+return result
+
+proc elna_lexer_parse_byte_literal(byte_literal: ^const Word8) -> StdOptional#[ElnaLexerToken]
var
buffer: [4]Word8
output := buffer.ptr
- current_position := escape + 1
+ current_position := byte_literal + 1
consumed := elna_lexer_decode_one_character(current_position, @output)
result := { StdOptional#[ElnaLexerToken] }
begin
@@ -1100,6 +1198,8 @@ begin
result := std_optional_make#[ElnaLexerToken](comment_token)
elsif start^ = '\'' then
result := elna_lexer_parse_byte_literal(start)
+ elsif start^ = '`' then
+ result := elna_lexer_parse_character_literal(start)
end
return result
@@ -1207,15 +1307,15 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
ElnaLexerTokenKind.generic_open: printf("«#[»\0".ptr) |
ElnaLexerTokenKind.attribute_open: printf("«#{»\0".ptr) |
ElnaLexerTokenKind.word:
- printf("«0x%xu»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+ printf("«0x%Xu»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
ElnaLexerTokenKind.word8:
- printf("«0x%02xu8»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+ printf("«0x%02Xu8»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
ElnaLexerTokenKind.word16:
- printf("«0x%04xu16»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+ printf("«0x%04Xu16»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
ElnaLexerTokenKind.word32:
- printf("«0x%08xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+ printf("«0x%08Xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
ElnaLexerTokenKind.word64:
- printf("«0x%016lxu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
+ printf("«0x%016lXu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
ElnaLexerTokenKind.integer: printf("integer\0".ptr) |
ElnaLexerTokenKind.integer8: printf("integer8\0".ptr) |
ElnaLexerTokenKind.integer16: printf("integer16\0".ptr) |
@@ -1223,7 +1323,8 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
ElnaLexerTokenKind.integer64: printf("integer64\0".ptr) |
ElnaLexerTokenKind.double: printf("double\0".ptr) |
ElnaLexerTokenKind.single: printf("single\0".ptr) |
- ElnaLexerTokenKind.character: printf("character\0".ptr) |
+ ElnaLexerTokenKind.character:
+ printf("\\{U+%X}\0".ptr, cast(token: ^ElnaLexerCharacterToken)^.codepoint) |
ElnaLexerTokenKind.string: printf("string\0".ptr) |
ElnaLexerTokenKind.left_paren: printf("«(»\0".ptr) |
ElnaLexerTokenKind.right_paren: printf("«)»\0".ptr) |
diff --git a/source/main.elna b/source/main.elna
index acc5674..7065b3b 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer
var
global_cursor: ElnaLexerCursor
- sample_source_text := "Here.are a few (* documentation block *) or if '\\n' -- End line comment\0"
+ sample_source_text := "Here.are a few (* documentation block *) or if '\\n' `У` -- End line comment\0"
token: ^ElnaLexerToken
begin
generated by cgit v1.2.3 (git 2.26.2) at 2026-11-02 12:43:07 +0000