aboutsummaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
Diffstat (limited to 'source')
-rw-r--r--source/elna/lexer.elna88
-rw-r--r--source/main.elna2
2 files changed, 51 insertions, 39 deletions
diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna
index 0c32973..d190526 100644
--- a/source/elna/lexer.elna
+++ b/source/elna/lexer.elna
@@ -5,7 +5,9 @@
(* Lexical analysis. *)
module
-import std, sys.posix.stdlib, sys.posix.string, sys.posix.stdio, elna.source
+import
+ std, sys.posix.stdlib, sys.posix.string, sys.posix.stdio, sys.posix.errno,
+ elna.source
type
(*
@@ -210,16 +212,7 @@ type
trait,
generic_open,
attribute_open,
- word,
- word8,
- word16,
- word32,
- word64,
integer,
- integer8,
- integer16,
- integer32,
- integer64,
double,
single,
character,
@@ -273,11 +266,10 @@ type
ElnaLexerCommentToken = record(ElnaLexerToken)
content: StdString
end
- ElnaLexerWordToken = record(ElnaLexerToken)
- value: Word64
- end
ElnaLexerIntegerToken = record(ElnaLexerToken)
- value: Int64
+ magnitude: Word64;
+ size: Word8; -- 0 for target Word/Int, or 8, 16, 32, 64 for explicit width.
+ is_signed: Bool
end
ElnaLexerFloatToken = record(ElnaLexerToken)
value: Double
@@ -1061,10 +1053,12 @@ begin
-- Unicode consumes more than 4 characters and is not supported here.
if consumed > 0 & consumed <= 4 & current_position^ = '\'' then
var
- byte_token := malloc#[ElnaLexerWordToken](#size(ElnaLexerWordToken))
+ byte_token := malloc#[ElnaLexerIntegerToken](#size(ElnaLexerIntegerToken))
begin
- byte_token^.kind := ElnaLexerTokenKind.word8;
- byte_token^.value := cast(buffer[1]: Word64);
+ byte_token^.kind := ElnaLexerTokenKind.integer;
+ byte_token^.magnitude := cast(buffer[1]: Word64);
+ byte_token^.is_signed := false;
+ byte_token^.size := 8u8;
result := std_optional_make#[ElnaLexerToken](byte_token)
end
@@ -1103,10 +1097,31 @@ return result
proc elna_lexer_parse_integer(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken]
var
number_string := start[1 to cursor - start]
+ parsed_number: Word64
+ number_stop: ^Word8
+ result: ^ElnaLexerIntegerToken
begin
+ errno()^ := 0i32;
+
if number_string.length > 2u & number_string[1] = '0' & (number_string[2] = 'b' or number_string[2] = 'B') then
+ parsed_number := strtoull(start + 2, @number_stop, 2)
+ elsif number_string.length > 2u & number_string[1] = '0' & (number_string[2] = 'x' or number_string[2] = 'X') then
+ parsed_number := strtoull(start + 2, @number_stop, 16)
+ else
+ parsed_number := strtoull(start, @number_stop, 10)
+ end;
+ if errno()^ <> ERANGE then
+ if cursor = number_stop or (cursor - number_stop = 1 & number_stop^ = 'i') then
+ end;
+ result := malloc#[ElnaLexerIntegerToken](#size(ElnaLexerIntegerToken));
+ result^.kind := ElnaLexerTokenKind.integer;
+ result^.magnitude := cast(parsed_number: Word64);
+ result^.is_signed := true;
+ result^.size := 0u8
+
+ -- printf("%p %p %p\n\0".ptr, start, cursor, number_stop)
end
-return { StdOptional#[ElnaLexerToken] }
+return std_optional_make#[ElnaLexerToken](result)
-- TODO: Numbers are not tokenized.
proc elna_lexer_parse_float(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken]
@@ -1376,6 +1391,22 @@ proc elna_lexer_dump_string(token: ^ElnaLexerStringToken)
putchar(cast('"': Int))
return
+proc elna_lexer_dump_integer(token: ^ElnaLexerIntegerToken)
+ if token^.is_signed then
+ if token^.size = 0u8 then
+ printf("«%lld»\0".ptr, token^.magnitude)
+ else
+ printf("«%lldi%d»\0".ptr, token^.magnitude, token^.size)
+ end
+ else
+ if token^.size = 0u8 then
+ printf("«0x%Xu»\0".ptr, token^.magnitude)
+ else
+ printf("«0x%Xu%d»\0".ptr, token^.magnitude, token^.size)
+ end
+ end
+return
+
-- TODO: Replace printf and other writing functions with a writer callback.
-- TODO: Cursor shouldn't be a parameter, only token.
proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
@@ -1435,26 +1466,7 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
|
ElnaLexerTokenKind.generic_open: printf("«#[»\0".ptr) |
ElnaLexerTokenKind.attribute_open: printf("«#{»\0".ptr) |
- ElnaLexerTokenKind.word:
- printf("«0x%Xu»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
- ElnaLexerTokenKind.word8:
- printf("«0x%02Xu8»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
- ElnaLexerTokenKind.word16:
- printf("«0x%04Xu16»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
- ElnaLexerTokenKind.word32:
- printf("«0x%08Xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
- ElnaLexerTokenKind.word64:
- printf("«0x%016lXu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
- ElnaLexerTokenKind.integer:
- printf("«%lld»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
- ElnaLexerTokenKind.integer8:
- printf("«%lldi8»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
- ElnaLexerTokenKind.integer16:
- printf("«%lldi16»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
- ElnaLexerTokenKind.integer32:
- printf("«%lldi32»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
- ElnaLexerTokenKind.integer64:
- printf("«%lldi64»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
+ ElnaLexerTokenKind.integer: elna_lexer_dump_integer(cast(token: ^ElnaLexerIntegerToken)) |
ElnaLexerTokenKind.double:
printf("«%g»\0".ptr, cast(token: ^ElnaLexerFloatToken)^.value) |
ElnaLexerTokenKind.single:
diff --git a/source/main.elna b/source/main.elna
index b2145c8..fb1ae86 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer
var
global_cursor: ElnaLexerCursor
- sample_source_text := "Here.a few (* documentation block *) or #size if '\\n' `У` \"некоторая строка\" -- End line comment\0"
+ sample_source_text := "Here.a few (* documentation block *) or #size if '\\n' `У` \"некоторая строка\" 23 -- End line comment\0"
token: ^ElnaLexerToken
begin