diff options
| author | Eugen Wissner <belka@caraus.de> | 2026-11-09 20:01:28 +0100 |
|---|---|---|
| committer | Eugen Wissner <belka@caraus.de> | 2026-11-09 20:01:28 +0100 |
| commit | 0b5bd8114255e04dae27a7528a3111d6923e5fbe (patch) | |
| tree | 805fab3cb7744597d74ec310c4df8b74cd55847c /source | |
| parent | afa516a3bae3a2dd4bb4a92cb12f3f6fcdcc8e5c (diff) | |
| download | elna-0b5bd8114255e04dae27a7528a3111d6923e5fbe.tar.gz | |
Tokenize numbers
Diffstat (limited to 'source')
| -rw-r--r-- | source/elna/lexer.elna | 91 | ||||
| -rw-r--r-- | source/sys/posix/stdlib.elna | 6 |
2 files changed, 77 insertions, 20 deletions
diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna index d190526..f7bb4f7 100644 --- a/source/elna/lexer.elna +++ b/source/elna/lexer.elna @@ -1093,42 +1093,92 @@ begin end return result --- TODO: Numbers are not tokenized. +proc elna_lexer_parse_integer_bit_size(start, cursor: ^const Word8; size: ^Word8) -> Bool +var + known_size := true +begin + case start[1 to cursor - start] of + "": size^ := 0u8 | + "8": size^ := 8u8 | + "16": size^ := 16u8 | + "32": size^ := 32u8 | + "64": size^ := 64u8 + else + known_size := false + end +return known_size + proc elna_lexer_parse_integer(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken] var number_string := start[1 to cursor - start] - parsed_number: Word64 number_stop: ^Word8 - result: ^ElnaLexerIntegerToken + result := { StdOptional#[ElnaLexerToken] } + integer_token := { ElnaLexerIntegerToken; kind: ElnaLexerTokenKind.integer } + has_valid_suffix := false + is_prefixed := false begin errno()^ := 0i32; if number_string.length > 2u & number_string[1] = '0' & (number_string[2] = 'b' or number_string[2] = 'B') then - parsed_number := strtoull(start + 2, @number_stop, 2) + start := start + 2; + integer_token.magnitude := strtoull(start, @number_stop, 2); + is_prefixed := true elsif number_string.length > 2u & number_string[1] = '0' & (number_string[2] = 'x' or number_string[2] = 'X') then - parsed_number := strtoull(start + 2, @number_stop, 16) + start := start + 2; + integer_token.magnitude := strtoull(start, @number_stop, 16); + is_prefixed := true else - parsed_number := strtoull(start, @number_stop, 10) + integer_token.magnitude := strtoull(start, @number_stop, 10) end; - if errno()^ <> ERANGE then - if cursor = number_stop or (cursor - number_stop = 1 & number_stop^ = 'i') then - end; - result := malloc#[ElnaLexerIntegerToken](#size(ElnaLexerIntegerToken)); - result^.kind := ElnaLexerTokenKind.integer; - result^.magnitude := cast(parsed_number: Word64); - result^.is_signed := true; - result^.size := 0u8 - - -- printf("%p %p %p\n\0".ptr, start, cursor, number_stop) + if number_stop > start & errno()^ <> ERANGE then + if cursor = number_stop then + integer_token.is_signed := ~is_prefixed; + integer_token.size := 0u8; + has_valid_suffix := true + elsif cursor - number_stop >= 1 & number_stop^ = 'u' then + integer_token.is_signed := false; + has_valid_suffix := elna_lexer_parse_integer_bit_size(number_stop + 1, cursor, @integer_token.size) + elsif cursor - number_stop > 1 & number_stop^ = 'i' & ~is_prefixed then + integer_token.is_signed := true; + has_valid_suffix := elna_lexer_parse_integer_bit_size(number_stop + 1, cursor, @integer_token.size) + end + end; + if has_valid_suffix then + var + token_pointer := malloc#[ElnaLexerIntegerToken](#size(ElnaLexerIntegerToken)) + begin + token_pointer^ := integer_token; + result := std_optional_make#[ElnaLexerToken](token_pointer) end -return std_optional_make#[ElnaLexerToken](result) +return result --- TODO: Numbers are not tokenized. proc elna_lexer_parse_float(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken] var - number_string := start[1 to cursor - start] + result := { StdOptional#[ElnaLexerToken] } + number_stop: ^Word8 + float_token := { ElnaLexerFloatToken } + expected_stop: ^const Word8 begin -return { StdOptional#[ElnaLexerToken] } + errno()^ := 0i32; + + if (cursor - 1)^ = 'f' then + float_token.kind := ElnaLexerTokenKind.single; + expected_stop := cursor - 1; + float_token.value := cast(strtof(start, @number_stop): Double) + else + float_token.kind := ElnaLexerTokenKind.double; + expected_stop := cursor; + float_token.value := strtod(start, @number_stop) + end; + + if number_stop = expected_stop & errno()^ <> ERANGE then + var + float_pointer := malloc#[ElnaLexerFloatToken](#size(ElnaLexerFloatToken)) + begin + float_pointer^ := float_token; + result := std_optional_make#[ElnaLexerToken](float_pointer) + end +return result (* Skips blanks and advances the cursor to span the next significant token. @@ -1303,6 +1353,7 @@ begin return result -- TODO: Return type should be "error or a pointer". +-- TODO: Save the position in the token. proc elna_lexer_lex*(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken var result: ^ElnaLexerToken := cast(0: ^ElnaLexerToken) diff --git a/source/sys/posix/stdlib.elna b/source/sys/posix/stdlib.elna index 568e1da..497e87d 100644 --- a/source/sys/posix/stdlib.elna +++ b/source/sys/posix/stdlib.elna @@ -21,4 +21,10 @@ extern proc strtoull*(str: ^const Word8; endptr: ^^Word8; base: Int) -> Word64 extern +proc strtof*(nptr: ^const Word8; endptr: ^^Word8) -> Single +extern + +proc strtod*(nptr: ^const Word8; endptr: ^^Word8) -> Double +extern + end. |
