From 4b7c87a9b53f5317e9f7982bd779d53a66960bdc Mon Sep 17 00:00:00 2001
From: Eugen Wissner <belka@caraus.de>
Date: Sun, 16 Aug 2026 00:58:38 +0200
Subject: Support UTF-8 strings

---
 boot/lexer.ll | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

(limited to 'boot/lexer.ll')

diff --git a/boot/lexer.ll b/boot/lexer.ll
index 691034a..c6d3b17 100644
--- a/boot/lexer.ll
+++ b/boot/lexer.ll
@@ -351,22 +351,28 @@ to {
         return yy::parser::make_FLOAT(result, this->location);
     }
 }
-'([[:print:]]{-}['\\]|[[:blank:]]|\n|\r|\\.|\\\n)+' {
-    std::optional<std::string> result = escape_string(yytext);
-    if (!result.has_value() || result.value().size() != 1)
+'([^'\\\n]|\\.|\\\n)+' {
+    std::optional<std::uint8_t> result = parse_byte_literal(yytext);
+    if (!result.has_value())
+    {
+        REJECT;
+    }
+    return yy::parser::make_WORD8(std::make_pair(result.value(), integer_sign::_unsigned), this->location);
+}
+`([^'\\\n]|\\.|\\\n)+` {
+    std::optional<std::uint32_t> result = parse_character_literal(yytext);
+    if (!result.has_value())
     {
         REJECT;
     }
-    normalize_newlines(result.value());
     return yy::parser::make_CHARACTER(result.value(), this->location);
 }
-\"([[:print:]]{-}[\"\\]|[[:blank:]]|\n|\r|\\.|\\\n)*\" {
-    std::optional<std::string> result = escape_string(yytext);
+\"([^\"\\\n]|\\.|\\\n)*\" {
+    std::optional<std::string> result = parse_string_literal(yytext);
     if (!result.has_value())
     {
         REJECT;
     }
-    normalize_newlines(result.value());
     return yy::parser::make_STRING(result.value(), this->location);
 }
 \( {
-- 
cgit v1.2.3