aboutsummaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
Diffstat (limited to 'source')
-rw-r--r--source/elna/lexer.elna196
-rw-r--r--source/main.elna2
-rw-r--r--source/sys/posix/errno.elna8
-rw-r--r--source/sys/posix/stdio.elna6
-rw-r--r--source/sys/posix/stdlib.elna3
-rw-r--r--source/sys/posix/string.elna4
6 files changed, 168 insertions, 51 deletions
diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna
index 0a72d0d..0c32973 100644
--- a/source/elna/lexer.elna
+++ b/source/elna/lexer.elna
@@ -129,8 +129,10 @@ type
integer,
(* Inside a comment (after eating double minus, --). *)
comment,
- (* Tail is the last part of the number containing digits and an optional type suffix. *)
- number_tail,
+ (* Tail is the last part of the integer containing digits and an optional type suffix. *)
+ integer_tail,
+ (* Tail is the last part of the floating point number containing digits and an optional type suffix. *)
+ float_tail,
(* Entering floating point number fraction part after a decimal point. *)
fraction,
(* An exponent state begins with e or E. *)
@@ -274,6 +276,12 @@ type
ElnaLexerWordToken = record(ElnaLexerToken)
value: Word64
end
+ ElnaLexerIntegerToken = record(ElnaLexerToken)
+ value: Int64
+ end
+ ElnaLexerFloatToken = record(ElnaLexerToken)
+ value: Double
+ end
ElnaLexerCharacterToken = record(ElnaLexerToken)
codepoint: Word32
end
@@ -658,14 +666,14 @@ begin
-- Numbers prefixed by 0 followed by a character (0b, 0x).
elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.zero, ElnaLexerState.integer);
elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.digit, ElnaLexerState.integer);
- elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.alpha, ElnaLexerState.number_tail);
+ elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.alpha, ElnaLexerState.integer_tail);
elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.exponent, ElnaLexerState.exponent);
elna_lexer_set_transition(ElnaLexerState.leading_zero, ElnaLexerClass.dot, ElnaLexerState.dot_after);
-- Decimal number.
elna_lexer_set_transition(ElnaLexerState.integer, ElnaLexerClass.zero, ElnaLexerState.integer);
elna_lexer_set_transition(ElnaLexerState.integer, ElnaLexerClass.digit, ElnaLexerState.integer);
- elna_lexer_set_transition(ElnaLexerState.integer, ElnaLexerClass.alpha, ElnaLexerState.number_tail);
+ elna_lexer_set_transition(ElnaLexerState.integer, ElnaLexerClass.alpha, ElnaLexerState.integer_tail);
elna_lexer_set_transition(ElnaLexerState.integer, ElnaLexerClass.exponent, ElnaLexerState.exponent);
elna_lexer_set_transition(ElnaLexerState.integer, ElnaLexerClass.dot, ElnaLexerState.dot_after);
@@ -687,11 +695,17 @@ begin
elna_lexer_set_transition(ElnaLexerState.closing_docblock, ElnaLexerClass.invalid, ElnaLexerState.reject);
elna_lexer_set_transition(ElnaLexerState.closing_docblock, ElnaLexerClass.eof, ElnaLexerState.reject);
- -- Tail is the last part of the number containing digits and an optional type suffix.
- elna_lexer_set_transition(ElnaLexerState.number_tail, ElnaLexerClass.exponent, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.number_tail, ElnaLexerClass.alpha, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.number_tail, ElnaLexerClass.zero, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.number_tail, ElnaLexerClass.digit, ElnaLexerState.number_tail);
+ -- Tail is the last part of the integer containing digits and an optional type suffix.
+ elna_lexer_set_transition(ElnaLexerState.integer_tail, ElnaLexerClass.exponent, ElnaLexerState.integer_tail);
+ elna_lexer_set_transition(ElnaLexerState.integer_tail, ElnaLexerClass.alpha, ElnaLexerState.integer_tail);
+ elna_lexer_set_transition(ElnaLexerState.integer_tail, ElnaLexerClass.zero, ElnaLexerState.integer_tail);
+ elna_lexer_set_transition(ElnaLexerState.integer_tail, ElnaLexerClass.digit, ElnaLexerState.integer_tail);
+
+ -- Tail is the last part of the floating point number containing digits and an optional type suffix.
+ elna_lexer_set_transition(ElnaLexerState.float_tail, ElnaLexerClass.exponent, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.float_tail, ElnaLexerClass.alpha, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.float_tail, ElnaLexerClass.zero, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.float_tail, ElnaLexerClass.digit, ElnaLexerState.float_tail);
-- Seen a period after a number, possible floating point number.
elna_lexer_set_transition(ElnaLexerState.dot_after, ElnaLexerClass.zero, ElnaLexerState.fraction);
@@ -701,21 +715,21 @@ begin
elna_lexer_set_transition(ElnaLexerState.fraction, ElnaLexerClass.zero, ElnaLexerState.fraction);
elna_lexer_set_transition(ElnaLexerState.fraction, ElnaLexerClass.digit, ElnaLexerState.fraction);
elna_lexer_set_transition(ElnaLexerState.fraction, ElnaLexerClass.exponent, ElnaLexerState.exponent);
- elna_lexer_set_transition(ElnaLexerState.fraction, ElnaLexerClass.alpha, ElnaLexerState.number_tail);
+ elna_lexer_set_transition(ElnaLexerState.fraction, ElnaLexerClass.alpha, ElnaLexerState.float_tail);
-- An exponent state begins with e or E.
elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.plus, ElnaLexerState.exponent_sign);
elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.minus, ElnaLexerState.exponent_sign);
- elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.zero, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.digit, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.exponent, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.alpha, ElnaLexerState.number_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.zero, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.digit, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.exponent, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent, ElnaLexerClass.alpha, ElnaLexerState.float_tail);
-- Optional plus or minus sign following the exponent.
- elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.zero, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.digit, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.exponent, ElnaLexerState.number_tail);
- elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.alpha, ElnaLexerState.number_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.zero, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.digit, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.exponent, ElnaLexerState.float_tail);
+ elna_lexer_set_transition(ElnaLexerState.exponent_sign, ElnaLexerClass.alpha, ElnaLexerState.float_tail);
-- Attribute, generic parameter or trait marker ('#').
elna_lexer_set_transition(ElnaLexerState.number_sign, ElnaLexerClass.exponent, ElnaLexerState.trait);
@@ -1056,7 +1070,6 @@ begin
end
return result
--- TODO: Support dumping and test.
proc elna_lexer_parse_string_literal(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken]
var
raw_length := cast(cursor - start: Word)
@@ -1070,9 +1083,9 @@ begin
consumed := elna_lexer_decode_one_character(start, @output)
begin
if consumed = 0 then
- start := start + consumed
- else
failed := true
+ else
+ start := start + consumed
end
end;
if ~failed then
@@ -1081,10 +1094,27 @@ begin
final_length := cast(output - storage: Word)
begin
string_token^.kind := ElnaLexerTokenKind.string;
- string_token^.string := realloc(storage, final_length)[1 to final_length]
+ string_token^.string := realloc(storage, final_length)[1 to final_length];
+ result := std_optional_make#[ElnaLexerToken](string_token)
end
return result
+-- TODO: Numbers are not tokenized.
+proc elna_lexer_parse_integer(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken]
+var
+ number_string := start[1 to cursor - start]
+begin
+ if number_string.length > 2u & number_string[1] = '0' & (number_string[2] = 'b' or number_string[2] = 'B') then
+ end
+return { StdOptional#[ElnaLexerToken] }
+
+-- TODO: Numbers are not tokenized.
+proc elna_lexer_parse_float(start, cursor: ^const Word8) -> StdOptional#[ElnaLexerToken]
+var
+ number_string := start[1 to cursor - start]
+begin
+return { StdOptional#[ElnaLexerToken] }
+
(*
Skips blanks and advances the cursor to span the next significant token.
*)
@@ -1111,16 +1141,24 @@ return result
Frees the memory allocated for the token itself and for its internal storage.
*)
proc elna_lexer_token_destroy*(token: ^ElnaLexerToken)
- if token^.kind = ElnaLexerTokenKind.identifier then
- var
- identifier_token := cast(token: ^ElnaLexerIdentifierToken)
- begin
- free(identifier_token^.identifier.ptr)
- elsif token^.kind = ElnaLexerTokenKind.docblock or token^.kind = ElnaLexerTokenKind.comment then
- var
- comment_token := cast(token: ^ElnaLexerCommentToken)
- begin
- free(comment_token^.content.ptr)
+ case token^.kind of
+ ElnaLexerTokenKind.identifier, ElnaLexerTokenKind.trait:
+ var
+ identifier_token := cast(token: ^ElnaLexerIdentifierToken)
+ begin
+ free(identifier_token^.identifier.ptr)
+ |
+ ElnaLexerTokenKind.docblock, ElnaLexerTokenKind.comment:
+ var
+ comment_token := cast(token: ^ElnaLexerCommentToken)
+ begin
+ free(comment_token^.content.ptr)
+ |
+ ElnaLexerTokenKind.string:
+ var
+ string_token := cast(token: ^ElnaLexerStringToken)
+ begin
+ free(string_token^.string.ptr)
end;
free(token)
@@ -1129,10 +1167,8 @@ return
proc elna_lexer_identify_keyword(start, cursor: ^const Word8) -> ^ElnaLexerToken
var
result: ^ElnaLexerToken
- identifier: StdString
+ identifier: StdString := start[1 to cursor - start]
begin
- identifier := start[1 to cursor - start];
-
case identifier of
"if": result := elna_lexer_token_create(ElnaLexerTokenKind.if_) |
"then": result := elna_lexer_token_create(ElnaLexerTokenKind.then_) |
@@ -1170,7 +1206,7 @@ begin
"module": result := elna_lexer_token_create(ElnaLexerTokenKind.module_)
else
var
- identifier_token := malloc#[ElnaLexerIdentifierToken](#size(ElnaLexerToken))
+ identifier_token := malloc#[ElnaLexerIdentifierToken](#size(ElnaLexerIdentifierToken))
identifier_pointer := malloc#[const Word8](identifier.length)
begin
memcpy(identifier_pointer, identifier.ptr, identifier.length);
@@ -1181,6 +1217,18 @@ begin
end
return result
+proc elna_lexer_identify_trait(start, cursor: ^const Word8) -> ^ElnaLexerToken
+var
+ identifier: StdString := start[2 to cursor - start]
+ result := malloc#[ElnaLexerIdentifierToken](#size(ElnaLexerIdentifierToken))
+ identifier_pointer := malloc#[const Word8](identifier.length)
+begin
+ memcpy(identifier_pointer, identifier.ptr, identifier.length);
+
+ result^.kind := ElnaLexerTokenKind.trait;
+ result^.identifier := identifier_pointer[1 to identifier.length]
+return result
+
proc elna_lexer_stop_single(start: ^const Word8) -> ^ElnaLexerToken
var
result: ^ElnaLexerToken
@@ -1235,7 +1283,7 @@ begin
elsif start^ = '`' then
result := elna_lexer_parse_character_literal(start)
elsif start^ = '"' then
- result := elna_lexer_parse_string_literal(start + 1, cursor)
+ result := elna_lexer_parse_string_literal(start + 1, cursor - 1)
end
return result
@@ -1248,11 +1296,21 @@ begin
case cursor^.state of
ElnaLexerState.stop:
- result := elna_lexer_delimit_string(cursor^.start, cursor^.cursor - 2u).value
+ result := elna_lexer_delimit_string(cursor^.start, cursor^.cursor).value
|
ElnaLexerState.comment:
result := elna_lexer_stop_comment(ElnaLexerTokenKind.comment, cursor^.start, cursor^.cursor)
|
+ ElnaLexerState.leading_zero,
+ ElnaLexerState.integer,
+ ElnaLexerState.integer_tail:
+ result := elna_lexer_parse_integer(cursor^.start, cursor^.cursor).value
+ |
+ ElnaLexerState.float_tail,
+ ElnaLexerState.exponent,
+ ElnaLexerState.fraction:
+ result := elna_lexer_parse_float(cursor^.start, cursor^.cursor).value
+ |
ElnaLexerState.identifier: result := elna_lexer_identify_keyword(cursor^.start, cursor^.cursor) |
ElnaLexerState.single: result := elna_lexer_stop_single(cursor^.start) |
ElnaLexerState.attribute_open: result := elna_lexer_token_create(ElnaLexerTokenKind.attribute_open) |
@@ -1264,10 +1322,16 @@ begin
ElnaLexerState.not_equal: result := elna_lexer_token_create(ElnaLexerTokenKind.not_equal) |
ElnaLexerState.assignment: result := elna_lexer_token_create(ElnaLexerTokenKind.assignment) |
ElnaLexerState.arrow: result := elna_lexer_token_create(ElnaLexerTokenKind.arrow) |
+ ElnaLexerState.colon: result := elna_lexer_token_create(ElnaLexerTokenKind.colon) |
+ ElnaLexerState.left_paren: result := elna_lexer_token_create(ElnaLexerTokenKind.left_paren) |
+ ElnaLexerState.minus: result := elna_lexer_token_create(ElnaLexerTokenKind.minus) |
+ ElnaLexerState.greater: result := elna_lexer_token_create(ElnaLexerTokenKind.greater_than) |
+ ElnaLexerState.less: result := elna_lexer_token_create(ElnaLexerTokenKind.less_than) |
+ ElnaLexerState.trait: result := elna_lexer_identify_trait(cursor^.start, cursor^.cursor) |
ElnaLexerState.eof: result := elna_lexer_token_create(ElnaLexerTokenKind.eof)
else
result := malloc#[ElnaLexerToken](#size(ElnaLexerToken));
- memset(result, 0i8, #size(ElnaLexerToken))
+ memset(result, 0, #size(ElnaLexerToken))
end
return result
@@ -1288,6 +1352,30 @@ proc elna_lexer_token_dump_docblock(token: ^ElnaLexerCommentToken)
end
return
+proc elna_lexer_dump_string(token: ^ElnaLexerStringToken)
+ putchar(cast('"': Int));
+ for c of token^.string do
+ if c^ = '\f' then
+ printf("\\f\0".ptr)
+ elsif c^ = '\v' then
+ printf("\\v\0".ptr)
+ elsif c^ = '\\' then
+ printf("\\\\\0".ptr)
+ elsif c^ = '"' then
+ printf("\\\"\0".ptr)
+ elsif c^ = '\0' then
+ printf("\\0\0".ptr)
+ elsif c^ = '\n' or c^ = '\t' or c^ = '\r' then
+ putchar(cast(c^: Int))
+ elsif c^ < 32u8 or c^ = 127u8 then
+ printf("\\x%02X\0".ptr, c^)
+ else
+ putchar(cast(c^: Int))
+ end
+ end;
+ putchar(cast('"': Int))
+return
+
-- TODO: Replace printf and other writing functions with a writer callback.
-- TODO: Cursor shouldn't be a parameter, only token.
proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
@@ -1339,7 +1427,12 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
begin
printf("Identifier «%.*s»\0".ptr, identifier.length, identifier.ptr)
|
- ElnaLexerTokenKind.trait: printf("trait\0".ptr) |
+ ElnaLexerTokenKind.trait:
+ var
+ identifier := cast(token: ^ElnaLexerIdentifierToken)^.identifier
+ begin
+ printf("#%.*s\0".ptr, identifier.length, identifier.ptr)
+ |
ElnaLexerTokenKind.generic_open: printf("«#[»\0".ptr) |
ElnaLexerTokenKind.attribute_open: printf("«#{»\0".ptr) |
ElnaLexerTokenKind.word:
@@ -1352,16 +1445,23 @@ proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor)
printf("«0x%08Xu32»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
ElnaLexerTokenKind.word64:
printf("«0x%016lXu64»\0".ptr, cast(token: ^ElnaLexerWordToken)^.value) |
- ElnaLexerTokenKind.integer: printf("integer\0".ptr) |
- ElnaLexerTokenKind.integer8: printf("integer8\0".ptr) |
- ElnaLexerTokenKind.integer16: printf("integer16\0".ptr) |
- ElnaLexerTokenKind.integer32: printf("integer32\0".ptr) |
- ElnaLexerTokenKind.integer64: printf("integer64\0".ptr) |
- ElnaLexerTokenKind.double: printf("double\0".ptr) |
- ElnaLexerTokenKind.single: printf("single\0".ptr) |
+ ElnaLexerTokenKind.integer:
+ printf("«%lld»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
+ ElnaLexerTokenKind.integer8:
+ printf("«%lldi8»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
+ ElnaLexerTokenKind.integer16:
+ printf("«%lldi16»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
+ ElnaLexerTokenKind.integer32:
+ printf("«%lldi32»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
+ ElnaLexerTokenKind.integer64:
+ printf("«%lldi64»\0".ptr, cast(token: ^ElnaLexerIntegerToken)^.value) |
+ ElnaLexerTokenKind.double:
+ printf("«%g»\0".ptr, cast(token: ^ElnaLexerFloatToken)^.value) |
+ ElnaLexerTokenKind.single:
+ printf("«%gf»\0".ptr, cast(token: ^ElnaLexerFloatToken)^.value) |
ElnaLexerTokenKind.character:
printf("\\{U+%X}\0".ptr, cast(token: ^ElnaLexerCharacterToken)^.codepoint) |
- ElnaLexerTokenKind.string: printf("string\0".ptr) |
+ ElnaLexerTokenKind.string: elna_lexer_dump_string(cast(token: ^ElnaLexerStringToken)) |
ElnaLexerTokenKind.left_paren: printf("«(»\0".ptr) |
ElnaLexerTokenKind.right_paren: printf("«)»\0".ptr) |
ElnaLexerTokenKind.left_square: printf("«[»\0".ptr) |
diff --git a/source/main.elna b/source/main.elna
index 7065b3b..b2145c8 100644
--- a/source/main.elna
+++ b/source/main.elna
@@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer
var
global_cursor: ElnaLexerCursor
- sample_source_text := "Here.are a few (* documentation block *) or if '\\n' `У` -- End line comment\0"
+ sample_source_text := "Here.a few (* documentation block *) or #size if '\\n' `У` \"некоторая строка\" -- End line comment\0"
token: ^ElnaLexerToken
begin
diff --git a/source/sys/posix/errno.elna b/source/sys/posix/errno.elna
new file mode 100644
index 0000000..7035ddf
--- /dev/null
+++ b/source/sys/posix/errno.elna
@@ -0,0 +1,8 @@
+(* This Source Code Form is subject to the terms of the Mozilla Public License,
+ v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ obtain one at https://mozilla.org/MPL/2.0/. *)
+module
+
+import bits.errno*
+
+end.
diff --git a/source/sys/posix/stdio.elna b/source/sys/posix/stdio.elna
index dd7251d..39756a8 100644
--- a/source/sys/posix/stdio.elna
+++ b/source/sys/posix/stdio.elna
@@ -6,4 +6,10 @@ module
proc printf*(format: ^const Word8) #{varargs} -> Int
extern
+proc puts*(s: ^const Word8) -> Int
+extern
+
+proc putchar*(c: Int) -> Int
+extern
+
end.
diff --git a/source/sys/posix/stdlib.elna b/source/sys/posix/stdlib.elna
index d4b381e..568e1da 100644
--- a/source/sys/posix/stdlib.elna
+++ b/source/sys/posix/stdlib.elna
@@ -18,4 +18,7 @@ extern
proc strtoul*(str: ^const Word8; endptr: ^^Word8; base: Int) -> Word
extern
+proc strtoull*(str: ^const Word8; endptr: ^^Word8; base: Int) -> Word64
+extern
+
end.
diff --git a/source/sys/posix/string.elna b/source/sys/posix/string.elna
index ebe347c..561d975 100644
--- a/source/sys/posix/string.elna
+++ b/source/sys/posix/string.elna
@@ -3,13 +3,13 @@
obtain one at https://mozilla.org/MPL/2.0/. *)
module
-proc memset#[T]*(b: ^T; c: Int8; len: Word) -> ^T
+proc memset#[T]*(b: ^T; c: Int; len: Word) -> ^T
extern
proc memcpy#[T]*(dst: ^T; src: ^const T; n: Word) -> ^T
extern
-proc memchr#[T]*(s: ^T; c: Int8; n: Word) -> ^T
+proc memchr#[T]*(s: ^T; c: Int; n: Word) -> ^T
extern
end.