From 9f85fa784927f5b6ff07ce043eb46a3e9236760f Mon Sep 17 00:00:00 2001 From: Eugen Wissner Date: Wed, 21 Oct 2026 15:53:41 +0200 Subject: Allow any word type for shift operators --- source/elna/lexer.elna | 201 ++++++++++++++++++++++++++++++++++++++++++++++++- source/main.elna | 18 ++--- 2 files changed, 207 insertions(+), 12 deletions(-) (limited to 'source') diff --git a/source/elna/lexer.elna b/source/elna/lexer.elna index 4de178e..1130b7a 100644 --- a/source/elna/lexer.elna +++ b/source/elna/lexer.elna @@ -5,7 +5,7 @@ (* Lexical analysis. *) module -import std, sys.posix.stdlib, sys.posix.string, elna.source +import std, sys.posix.stdlib, sys.posix.string, sys.posix.stdio, elna.source type (* @@ -264,6 +264,22 @@ type ElnaLexerIdentifierToken = record(ElnaLexerToken) identifier: StdString end + ElnaLexerCommentToken = record(ElnaLexerToken) + content: StdString + end + +const + ELNA_LEXER_UTF8_LEAD_TAG_2BYTE = 0xC0u8 + ELNA_LEXER_UTF8_LEAD_TAG_3BYTE = 0xE0u8 + ELNA_LEXER_UTF8_LEAD_TAG_4BYTE = 0xF0u8 + + ELNA_LEXER_UTF8_CONTINUATION_BITS = 6u32 + ELNA_LEXER_UTF8_CONTINUATION_MASK = 0x3Fu32 + ELNA_LEXER_UTF8_CONTINUATION_TAG = 0x80u8 + + ELNA_LEXER_UTF8_SURROGATE_RANGE_START = 0xD800u32 + ELNA_LEXER_UTF8_SURROGATE_RANGE_END = 0xDFFFu32 + ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT = 0x10FFFFu32 var (* Initializes the array with character classes. *) @@ -764,6 +780,44 @@ proc elna_lexer_reset_cursor(cursor: ^ElnaLexerCursor) cursor^.start := cursor^.cursor return +proc elna_lexer_is_hex(character: Word8) -> Bool +return (character >= '0' & character <= '9') + or (character >= 'a' & character <= 'f') + or (character >= 'A' & character <= 'F') + +proc encode_utf8(codepoint: Word32; buffer: ^[4]Word8) -> Word +const + UTF8_1BYTE_MAX = 0x80u32 + UTF8_2BYTE_MAX = 0x800u32 + UTF8_3BYTE_MAX = 0x10000u32 +var + encoded: Word +begin + if (codepoint >= ELNA_LEXER_UTF8_SURROGATE_RANGE_START & codepoint <= ELNA_LEXER_UTF8_SURROGATE_RANGE_END) + or codepoint > ELNA_LEXER_UTF8_MAX_UNICODE_CODEPOINT then + encoded := 0u + elsif codepoint < UTF8_1BYTE_MAX then + buffer^[1] := cast(codepoint: Word8); + encoded := 1u + elsif codepoint < UTF8_2BYTE_MAX then + buffer^[1] := ELNA_LEXER_UTF8_CONTINUATION_TAG or cast(codepoint & ELNA_LEXER_UTF8_CONTINUATION_MASK: Word8); + buffer^[2] := ELNA_LEXER_UTF8_LEAD_TAG_2BYTE or cast(codepoint >> ELNA_LEXER_UTF8_CONTINUATION_BITS: Word8); + encoded := 2u + elsif codepoint < UTF8_3BYTE_MAX then + for i of buffer^[1 to 2] do + i^ := ELNA_LEXER_UTF8_CONTINUATION_TAG or cast(codepoint & ELNA_LEXER_UTF8_CONTINUATION_MASK: Word8); + codepoint := codepoint >> ELNA_LEXER_UTF8_CONTINUATION_BITS + end; + buffer^[3] := ELNA_LEXER_UTF8_LEAD_TAG_3BYTE or cast(codepoint: Word8) + else + for i of buffer^[1 to 3] do + i^ := ELNA_LEXER_UTF8_CONTINUATION_TAG or cast(codepoint & ELNA_LEXER_UTF8_CONTINUATION_MASK: Word8); + codepoint := codepoint >> ELNA_LEXER_UTF8_CONTINUATION_BITS + end; + buffer^[4] := ELNA_LEXER_UTF8_LEAD_TAG_4BYTE or cast(codepoint: Word8) + end +return encoded + (* Skips blanks and advances the cursor to span the next significant token. *) @@ -795,6 +849,11 @@ proc elna_lexer_token_destroy*(token: ^ElnaLexerToken) identifier_token := cast(token: ^ElnaLexerIdentifierToken) begin free(identifier_token^.identifier.ptr) + elsif token^.kind = ElnaLexerTokenKind.docblock or token^.kind = ElnaLexerTokenKind.comment then + var + comment_token := cast(token: ^ElnaLexerCommentToken) + begin + free(comment_token^.content.ptr) end; free(token) @@ -850,7 +909,7 @@ begin memcpy(identifier_pointer, identifier.ptr, identifier.length); identifier_token^.kind := ElnaLexerTokenKind.identifier; - identifier_token^.identifier := identifier_pointer[1 to identifier.length];; + identifier_token^.identifier := identifier_pointer[1 to identifier.length]; result := identifier_token end return result @@ -883,6 +942,18 @@ begin end return result +proc elna_lexer_stop_comment(kind: ElnaLexerTokenKind; start, cursor: ^const Word8) -> ^ElnaLexerCommentToken +var + comment_token := malloc#[ElnaLexerCommentToken](#size(ElnaLexerCommentToken)) + comment_start: ^const Word8 := start + 2u + comment_length := cast(cursor - comment_start: Word) + comment_pointer := malloc#[Word8](comment_length) +begin + comment_token^.kind := kind; + memcpy(comment_pointer, comment_start, comment_length); + comment_token^.content := comment_pointer[1 to comment_length] +return comment_token + -- TODO: Return type should be "error or a pointer". proc elna_lexer_lex*(cursor: ^ElnaLexerCursor) -> ^ElnaLexerToken var @@ -891,6 +962,12 @@ begin elna_lexer_skip_trivia(cursor); case cursor^.state of + ElnaLexerState.stop: + result := elna_lexer_stop_comment(ElnaLexerTokenKind.docblock, cursor^.start, cursor^.cursor - 2u) + | + ElnaLexerState.comment: + result := elna_lexer_stop_comment(ElnaLexerTokenKind.comment, cursor^.start, cursor^.cursor) + | ElnaLexerState.identifier: result := elna_lexer_identify_keyword(cursor^.start, cursor^.cursor) | ElnaLexerState.single: result := elna_lexer_stop_single(cursor^.start) | ElnaLexerState.attribute_open: result := elna_lexer_token_create(ElnaLexerTokenKind.attribute_open) | @@ -901,11 +978,129 @@ begin ElnaLexerState.less_equal: result := elna_lexer_token_create(ElnaLexerTokenKind.less_equal) | ElnaLexerState.not_equal: result := elna_lexer_token_create(ElnaLexerTokenKind.not_equal) | ElnaLexerState.assignment: result := elna_lexer_token_create(ElnaLexerTokenKind.assignment) | - ElnaLexerState.arrow: result := elna_lexer_token_create(ElnaLexerTokenKind.arrow) + ElnaLexerState.arrow: result := elna_lexer_token_create(ElnaLexerTokenKind.arrow) | + ElnaLexerState.eof: result := elna_lexer_token_create(ElnaLexerTokenKind.eof) else result := malloc#[ElnaLexerToken](#size(ElnaLexerToken)); memset(result, 0i8, #size(ElnaLexerToken)) end return result +proc elna_lexer_token_dump_comment(token: ^ElnaLexerCommentToken) + if token^.content.length > 6u then + printf("--%.*s…%.*s\0".ptr, 3, token^.content.ptr, 3, token^.content.ptr + token^.content.length - 3u) + else + printf("--%.*s\0".ptr, token^.content.length, token^.content.ptr) + end +return + +proc elna_lexer_token_dump_docblock(token: ^ElnaLexerCommentToken) + if token^.content.length > 6u then + printf("(*%.*s…%.*s*)\0".ptr, 3, token^.content.ptr, + 3, token^.content.ptr + token^.content.length - 3u) + else + printf("(*%.*s*)\0".ptr, token^.content.length, token^.content.ptr) + end +return + +-- TODO: Replace printf and other writing functions with a writer callback. +-- TODO: Cursor shouldn't be a parameter, only token. +proc elna_lexer_token_dump*(token: ^ElnaLexerToken; cursor: ^ElnaLexerCursor) + printf("⟨\0".ptr); + + case token^.kind of + ElnaLexerTokenKind.comment: elna_lexer_token_dump_comment(cast(token: ^ElnaLexerCommentToken)) | + ElnaLexerTokenKind.docblock: elna_lexer_token_dump_docblock(cast(token: ^ElnaLexerCommentToken)) | + ElnaLexerTokenKind.if_: printf("«if»\0".ptr) | + ElnaLexerTokenKind.then_: printf("«then»\0".ptr) | + ElnaLexerTokenKind.else_: printf("«else»\0".ptr) | + ElnaLexerTokenKind.elsif_: printf("«elsif»\0".ptr) | + ElnaLexerTokenKind.while_: printf("«while»\0".ptr) | + ElnaLexerTokenKind.for_: printf("«for»\0".ptr) | + ElnaLexerTokenKind.repeat_: printf("«repeat»\0".ptr) | + ElnaLexerTokenKind.until_: printf("«until»\0".ptr) | + ElnaLexerTokenKind.with_: printf("«with»\0".ptr) | + ElnaLexerTokenKind.do_: printf("«do»\0".ptr) | + ElnaLexerTokenKind.proc_: printf("«proc»\0".ptr) | + ElnaLexerTokenKind.begin_: printf("«begin»\0".ptr) | + ElnaLexerTokenKind.end_: printf("«end»\0".ptr) | + ElnaLexerTokenKind.extern_: printf("«extern»\0".ptr) | + ElnaLexerTokenKind.const_: printf("«const»\0".ptr) | + ElnaLexerTokenKind.var_: printf("«var»\0".ptr) | + ElnaLexerTokenKind.threadvar_: printf("«threadvar»\0".ptr) | + ElnaLexerTokenKind.type_: printf("«type»\0".ptr) | + ElnaLexerTokenKind.record_: printf("«record»\0".ptr) | + ElnaLexerTokenKind.true_: printf("«true»\0".ptr) | + ElnaLexerTokenKind.false_: printf("«false»\0".ptr) | + ElnaLexerTokenKind.and: printf("«&»\0".ptr) | + ElnaLexerTokenKind.xor_: printf("«xor»\0".ptr) | + ElnaLexerTokenKind.or_: printf("«or»\0".ptr) | + ElnaLexerTokenKind.pipe: printf("«|»\0".ptr) | + ElnaLexerTokenKind.not: printf("«~»\0".ptr) | + ElnaLexerTokenKind.return_: printf("«return»\0".ptr) | + ElnaLexerTokenKind.import_: printf("«import»\0".ptr) | + ElnaLexerTokenKind.cast_: printf("«cast»\0".ptr) | + ElnaLexerTokenKind.defer_: printf("«defer»\0".ptr) | + ElnaLexerTokenKind.block_: printf("«block»\0".ptr) | + ElnaLexerTokenKind.break_: printf("«break»\0".ptr) | + ElnaLexerTokenKind.case_: printf("«case»\0".ptr) | + ElnaLexerTokenKind.of_: printf("«of»\0".ptr) | + ElnaLexerTokenKind.to_: printf("«to»\0".ptr) | + ElnaLexerTokenKind.program_: printf("«program»\0".ptr) | + ElnaLexerTokenKind.module_: printf("«module»\0".ptr) | + ElnaLexerTokenKind.identifier: + var + identifier := cast(token: ^ElnaLexerIdentifierToken)^.identifier + begin + printf("Identifier «%.*s»\0".ptr, identifier.length, identifier.ptr) + | + ElnaLexerTokenKind.trait: printf("trait\0".ptr) | + ElnaLexerTokenKind.generic_open: printf("«#[»\0".ptr) | + ElnaLexerTokenKind.attribute_open: printf("«#{»\0".ptr) | + ElnaLexerTokenKind.word: printf("word\0".ptr) | + ElnaLexerTokenKind.word8: printf("word8\0".ptr) | + ElnaLexerTokenKind.word16: printf("word16\0".ptr) | + ElnaLexerTokenKind.word32: printf("word32\0".ptr) | + ElnaLexerTokenKind.word64: printf("word64\0".ptr) | + ElnaLexerTokenKind.integer: printf("integer\0".ptr) | + ElnaLexerTokenKind.integer8: printf("integer8\0".ptr) | + ElnaLexerTokenKind.integer16: printf("integer16\0".ptr) | + ElnaLexerTokenKind.integer32: printf("integer32\0".ptr) | + ElnaLexerTokenKind.integer64: printf("integer64\0".ptr) | + ElnaLexerTokenKind.double: printf("double\0".ptr) | + ElnaLexerTokenKind.single: printf("single\0".ptr) | + ElnaLexerTokenKind.character: printf("character\0".ptr) | + ElnaLexerTokenKind.string: printf("string\0".ptr) | + ElnaLexerTokenKind.left_paren: printf("«(»\0".ptr) | + ElnaLexerTokenKind.right_paren: printf("«)»\0".ptr) | + ElnaLexerTokenKind.left_square: printf("«[»\0".ptr) | + ElnaLexerTokenKind.right_square: printf("«]»\0".ptr) | + ElnaLexerTokenKind.left_brace: printf("«{»\0".ptr) | + ElnaLexerTokenKind.right_brace: printf("«}»\0".ptr) | + ElnaLexerTokenKind.shift_left: printf("«<<»\0".ptr) | + ElnaLexerTokenKind.shift_right: printf("«>>»\0".ptr) | + ElnaLexerTokenKind.greater_equal: printf("«>=»\0".ptr) | + ElnaLexerTokenKind.less_equal: printf("«<=»\0".ptr) | + ElnaLexerTokenKind.greater_than: printf("«>»\0".ptr) | + ElnaLexerTokenKind.less_than: printf("«<»\0".ptr) | + ElnaLexerTokenKind.not_equal: printf("«<>»\0".ptr) | + ElnaLexerTokenKind.equals: printf("«=»\0".ptr) | + ElnaLexerTokenKind.semicolon: printf("«;»\0".ptr) | + ElnaLexerTokenKind.dot: printf("«.»\0".ptr) | + ElnaLexerTokenKind.comma: printf("«,»\0".ptr) | + ElnaLexerTokenKind.plus: printf("«+»\0".ptr) | + ElnaLexerTokenKind.minus: printf("«-»\0".ptr) | + ElnaLexerTokenKind.multiplication: printf("«*»\0".ptr) | + ElnaLexerTokenKind.division: printf("«/»\0".ptr) | + ElnaLexerTokenKind.remainder: printf("«%»\0".ptr) | + ElnaLexerTokenKind.assignment: printf("«:=»\0".ptr) | + ElnaLexerTokenKind.colon: printf("«:»\0".ptr) | + ElnaLexerTokenKind.hat: printf("«^»\0".ptr) | + ElnaLexerTokenKind.at: printf("«@»\0".ptr) | + ElnaLexerTokenKind.arrow: printf("«->»\0".ptr) | + ElnaLexerTokenKind.eof: printf("EOF\0".ptr) + end; + printf("; Position: %lu⟩\n\0".ptr, cursor^.position) +return + end. diff --git a/source/main.elna b/source/main.elna index fb58766..2099cb3 100644 --- a/source/main.elna +++ b/source/main.elna @@ -6,7 +6,7 @@ import sys.posix.stdio, elna.lexer var global_cursor: ElnaLexerCursor - sample_source_text := "Here.are a few 'tokens' or if\0" + sample_source_text := "Here.are a few (* documentation block *) or if -- End line comment\0" token: ^ElnaLexerToken begin @@ -17,14 +17,14 @@ begin position: 1u }; - repeat - token := elna_lexer_lex(@global_cursor); + token := elna_lexer_lex(@global_cursor); + elna_lexer_token_dump(token, @global_cursor); - printf("Token: %.*s, length: %lu, final state: %i, position: %lu, kind: %i\n\0".ptr, - global_cursor.cursor - global_cursor.start, global_cursor.start, - global_cursor.cursor - global_cursor.start, global_cursor.state, - global_cursor.position, token^.kind); + while token^.kind <> ElnaLexerTokenKind.eof do + elna_lexer_token_destroy(token); - elna_lexer_token_destroy(token) - until global_cursor.state = ElnaLexerState.eof + token := elna_lexer_lex(@global_cursor); + elna_lexer_token_dump(token, @global_cursor) + end; + elna_lexer_token_destroy(token) end. -- cgit v1.2.3