Support UTF-8 strings

1 files changed, 13 insertions, 7 deletions

diff --git a/boot/lexer.ll b/boot/lexer.ll
index 691034a..c6d3b17 100644
--- a/boot/lexer.ll
+++ b/boot/lexer.ll
@@ -351,22 +351,28 @@ to {
         return yy::parser::make_FLOAT(result, this->location);
     }
 }
-'([[:print:]]{-}['\\]|[[:blank:]]|\n|\r|\\.|\\\n)+' {
-    std::optional<std::string> result = escape_string(yytext);
-    if (!result.has_value() || result.value().size() != 1)
+'([^'\\\n]|\\.|\\\n)+' {
+    std::optional<std::uint8_t> result = parse_byte_literal(yytext);
+    if (!result.has_value())
+    {
+        REJECT;
+    }
+    return yy::parser::make_WORD8(std::make_pair(result.value(), integer_sign::_unsigned), this->location);
+}
+`([^'\\\n]|\\.|\\\n)+` {
+    std::optional<std::uint32_t> result = parse_character_literal(yytext);
+    if (!result.has_value())
     {
         REJECT;
     }
-    normalize_newlines(result.value());
     return yy::parser::make_CHARACTER(result.value(), this->location);
 }
-\"([[:print:]]{-}[\"\\]|[[:blank:]]|\n|\r|\\.|\\\n)*\" {
-    std::optional<std::string> result = escape_string(yytext);
+\"([^\"\\\n]|\\.|\\\n)*\" {
+    std::optional<std::string> result = parse_string_literal(yytext);
     if (!result.has_value())
     {
         REJECT;
     }
-    normalize_newlines(result.value());
     return yy::parser::make_STRING(result.value(), this->location);
 }
 \( {