diff options
| author | Eugen Wissner <belka@caraus.de> | 2026-11-01 22:10:48 +0100 |
|---|---|---|
| committer | Eugen Wissner <belka@caraus.de> | 2026-11-01 22:10:48 +0100 |
| commit | 48a51628b841a9b69db846326dcaa67239a8559f (patch) | |
| tree | 76707565b96d42ee652179e4037e42a2c5559888 /source/elna | |
| parent | 5dd5d460f154c146d26f115706a21b7808658edf (diff) | |
| download | elna-cpp.tar.gz | |
Diffstat (limited to 'source/elna')
| -rw-r--r-- | source/elna/lexer.elna | 38 |
1 files changed, 37 insertions, 1 deletions
diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna index a543c67..0a72d0d 100644 --- a/source/elna/lexer.elna +++ b/source/elna/lexer.elna @@ -277,6 +277,9 @@ type ElnaLexerCharacterToken = record(ElnaLexerToken) codepoint: Word32 end + ElnaLexerStringToken = record(ElnaLexerToken) + string: StdString + end const ELNA_LEXER_UTF8_LEAD_TAG_2BYTE = 0xC0u8 @@ -975,10 +978,12 @@ begin if escape <> ELNA_LEXER_ESCAPE_INVALID_CHAR then current_position := current_position + 2; output^^ := escape; + output^ := output^ + 1; consumed := 2 end else output^^ := current_position^; + output^ := output^ + 1; consumed := 1 end return consumed @@ -1051,6 +1056,35 @@ begin end return result +-- TODO: Support dumping and test. +proc elna_lexer_parse_string_literal(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken] +var + raw_length := cast(cursor - start: Word) + storage: ^Word8 := malloc#[Word8](raw_length) + output := storage + result := { StdOptional#[ElnaLexerToken] } + failed := false +begin + while ~failed & start < cursor do + var + consumed := elna_lexer_decode_one_character(start, @output) + begin + if consumed = 0 then + start := start + consumed + else + failed := true + end + end; + if ~failed then + var + string_token := malloc#[ElnaLexerStringToken](#size(ElnaLexerStringToken)) + final_length := cast(output - storage: Word) + begin + string_token^.kind := ElnaLexerTokenKind.string; + string_token^.string := realloc(storage, final_length)[1 to final_length] + end +return result + (* Skips blanks and advances the cursor to span the next significant token. *) @@ -1193,13 +1227,15 @@ var begin if start^ = '(' then var - comment_token := elna_lexer_stop_comment(ElnaLexerTokenKind.docblock, start, cursor - 2u) + comment_token := elna_lexer_stop_comment(ElnaLexerTokenKind.docblock, start, cursor - 2) begin result := std_optional_make#[ElnaLexerToken](comment_token) elsif start^ = '\'' then result := elna_lexer_parse_byte_literal(start) elsif start^ = '`' then result := elna_lexer_parse_character_literal(start) + elsif start^ = '"' then + result := elna_lexer_parse_string_literal(start + 1, cursor) end return result |
