From 4b7c87a9b53f5317e9f7982bd779d53a66960bdc Mon Sep 17 00:00:00 2001 From: Eugen Wissner Date: Sun, 16 Aug 2026 00:58:38 +0200 Subject: Support UTF-8 strings --- boot/driver.cc | 304 ++++++++++++++++++++++---- boot/lexer.ll | 20 +- boot/parser.yy | 4 +- include/elna/boot/driver.h | 28 ++- testsuite/runnable/unicode_escape_string.elna | 4 + 5 files changed, 307 insertions(+), 53 deletions(-) create mode 100644 testsuite/runnable/unicode_escape_string.elna diff --git a/boot/driver.cc b/boot/driver.cc index 3607995..2bcbcb1 100644 --- a/boot/driver.cc +++ b/boot/driver.cc @@ -17,6 +17,8 @@ along with GCC; see the file COPYING3. If not see #include "elna/boot/driver.h" +#include + namespace elna::boot { source_position make_position(const yy::location& location) @@ -39,7 +41,7 @@ namespace elna::boot return message; } - char escape_char(char escape) + static char unescape_character(char escape) { switch (escape) { @@ -66,68 +68,294 @@ namespace elna::boot } } - void normalize_newlines(std::string& string) + static bool is_hex(char character) + { + return (character >= '0' && character <= '9') + || (character >= 'a' && character <= 'f') + || (character >= 'A' && character <= 'F'); + } + + constexpr unsigned char utf8_lead_tag_2byte = 0xC0; + constexpr unsigned char utf8_lead_tag_3byte = 0xE0; + constexpr unsigned char utf8_lead_tag_4byte = 0xF0; + + constexpr int utf8_continuation_bits = 6; + constexpr uint32_t utf8_continuation_mask = 0x3F; + constexpr unsigned char utf8_continuation_tag = 0x80; + + constexpr uint32_t utf8_surrogate_range_start = 0xD800; + constexpr uint32_t utf8_surrogate_range_end = 0xDFFF; + constexpr uint32_t utf8_max_unicode_codepoint = 0x10FFFF; + + static std::optional encode_utf8(uint32_t codepoint) { + constexpr uint32_t utf8_1byte_max = 0x80; + constexpr uint32_t utf8_2byte_max = 0x800; + constexpr uint32_t utf8_3byte_max = 0x10000; + std::string result; - auto length = string.size(); - for (std::size_t i = 0; i < length; ++i) + if ((codepoint >= utf8_surrogate_range_start && codepoint <= utf8_surrogate_range_end) + || codepoint > utf8_max_unicode_codepoint) + { + return std::nullopt; + } + else if (codepoint < utf8_1byte_max) + { + result.push_back(static_cast(codepoint)); + } + else if (codepoint < utf8_2byte_max) + { + result.push_back(static_cast(utf8_continuation_tag | (codepoint & utf8_continuation_mask))); + result.push_back(static_cast(utf8_lead_tag_2byte | (codepoint >> utf8_continuation_bits))); + } + else if (codepoint < utf8_3byte_max) + { + for (int i = 0; i < 2; ++i) + { + result.push_back(static_cast(utf8_continuation_tag | (codepoint & utf8_continuation_mask))); + codepoint >>= utf8_continuation_bits; + } + result.push_back(static_cast(utf8_lead_tag_3byte | codepoint)); + } + else + { + for (int i = 0; i < 3; ++i) + { + result.push_back(static_cast(utf8_continuation_tag | (codepoint & utf8_continuation_mask))); + codepoint >>= utf8_continuation_bits; + } + result.push_back(static_cast(utf8_lead_tag_4byte | codepoint)); + } + return std::optional(result); + } + + static std::optional> decode_utf8(const char *current_position) + { + constexpr unsigned char lead_tag_4byte_test = 0xF8; + constexpr uint32_t lead_payload_mask_2byte = 0x1F; + constexpr uint32_t lead_payload_mask_3byte = 0x0F; + constexpr uint32_t lead_payload_mask_4byte = 0x07; + constexpr unsigned char ascii_bit_mask = 0x80; + + const unsigned char lead = static_cast(current_position[0]); + std::ptrdiff_t extra; + uint32_t codepoint; + + if ((lead & ascii_bit_mask) == 0x00) + { + extra = 0; + codepoint = lead; + } + else if ((lead & utf8_lead_tag_3byte) == utf8_lead_tag_2byte) + { + extra = 1; + codepoint = lead & lead_payload_mask_2byte; + } + else if ((lead & utf8_lead_tag_4byte) == utf8_lead_tag_3byte) + { + extra = 2; + codepoint = lead & lead_payload_mask_3byte; + } + else if ((lead & lead_tag_4byte_test) == utf8_lead_tag_4byte) + { + extra = 3; + codepoint = lead & lead_payload_mask_4byte; + } + else + { + return std::nullopt; + } + for (int i = 1; i <= extra; ++i) { - if (string[i] == '\r') + const unsigned char continuation_byte = static_cast(current_position[i]); + + if ((continuation_byte & utf8_lead_tag_2byte) != utf8_continuation_tag) { - if (i + 1 < length && string[i + 1] == '\n') - { - ++i; // Skip \n in \r\n pairs. - } - result.push_back('\n'); + return std::nullopt; + } + codepoint = (codepoint << utf8_continuation_bits) | (continuation_byte & utf8_continuation_mask); + } + return std::make_pair(codepoint, extra + 1); + } + + static std::optional> parse_codepoint_notation(const char *current_position) + { + if (current_position[0] != 'U' || current_position[1] != '+') + { + return std::nullopt; + } + current_position += 2; + std::ptrdiff_t consumed{ 2 }; + const char *digits_start = current_position; + + while (is_hex(*current_position)) + { + ++current_position; + } + constexpr std::size_t max_digits = 8; // Unicode maximum is U+10FFFF. + consumed += current_position - digits_start; + + if (current_position == digits_start || std::cmp_greater(consumed, max_digits)) + { + return std::nullopt; + } + const std::string hex_digits(digits_start, current_position); + const std::uint32_t codepoint = static_cast(std::stoul(hex_digits, nullptr, 16)); + + if ((codepoint >= utf8_surrogate_range_start && codepoint <= utf8_surrogate_range_end) + || codepoint > utf8_max_unicode_codepoint) + { + return std::nullopt; + } + return std::make_pair(codepoint, consumed); + } + + static std::ptrdiff_t decode_one_character(const char *current_position, std::string& output) + { + if (current_position[0] == '\\' && current_position[1] == 'x') + { + current_position += 2; + + if (!is_hex(current_position[0]) || !is_hex(current_position[1])) + { + return -1; + } + const std::string hex_digits(current_position, 2); + const char character = static_cast(std::stoi(hex_digits, nullptr, 16)); + + output.push_back(character); + + return 4; + } + else if (current_position[0] == '\\' && current_position[1] == '{') + { + current_position += 2; + auto unescaped = parse_codepoint_notation(current_position); + + if (!unescaped.has_value()) + { + return -1; + } + const auto [codepoint, consumed] = unescaped.value(); + + if (current_position[consumed] != '}') + { + return -1; + } + if (auto encoded = encode_utf8(codepoint)) + { + std::ranges::reverse_copy(encoded.value(), std::back_inserter(output)); + + return consumed + 3; } else { - result.push_back(string[i]); + return -1; + } + } + else if (*current_position == '\\') + { + ++current_position; + + const char escape = unescape_character(*current_position); + if (escape == escape_invalid_char) + { + return -1; } + output.push_back(escape); + return 2; + } + else + { + output.push_back(*current_position); + return 1; } - string = std::move(result); } - std::optional escape_string(const char *escape) + std::optional parse_character_literal(const char *escape) { - std::string result; const char *current_position = escape + 1; + std::uint32_t result{ 0 }; - while (*current_position != '\0') + if (current_position[0] == '\\' && current_position[1] == '{') { - if (*current_position == '\\' && (*(current_position + 1) == 'x' || *(current_position + 1) == 'X')) + current_position += 2; + auto unescaped = parse_codepoint_notation(current_position); + + if (!unescaped.has_value()) { - current_position += 2; + return std::nullopt; + } + const auto [codepoint, consumed] = unescaped.value(); - std::size_t processed; - char const character = static_cast(std::stoi(current_position, &processed, 16)); - if (processed == 0) - { - return std::nullopt; - } - else - { - current_position += processed - 1; - result.push_back(character); - } + if (current_position[consumed] != '}') + { + return std::nullopt; } - else if (*current_position == '\\') + result = codepoint; + current_position += consumed + 1; + } + else if (current_position[0] == '\\') + { + char const escape = unescape_character(current_position[1]); + + if (escape == escape_invalid_char) { - ++current_position; + return std::nullopt; + } + current_position += 2; + result = static_cast(static_cast(escape)); + } + else + { + if (auto decoded = decode_utf8(current_position)) + { + result = decoded->first; + current_position += decoded->second; + } + else + { + return std::nullopt; + } + } + return *current_position == '`' ? std::optional(result) : std::nullopt; + } - char const escape = escape_char(*current_position); - if (escape == escape_invalid_char) - { - return std::nullopt; - } - result.push_back(escape); + std::optional parse_byte_literal(const char *escape) + { + std::string result; + const char *current_position = escape + 1; + const ptrdiff_t consumed = decode_one_character(current_position, result); + + // Unicode consumes more than 4 characters and is not supported here. + if (consumed == -1 || consumed > 4 || current_position[consumed] != '\'') + { + return std::nullopt; + } + else + { + return static_cast(result.at(0)); + } + } + + std::optional parse_string_literal(const char *escape) + { + std::string result; + const char *current_position = escape + 1; + + while (*current_position != '\0') + { + const ptrdiff_t consumed = decode_one_character(current_position, result); + + if (consumed == -1) + { + return std::nullopt; } else { - result.push_back(*current_position); + current_position += consumed; } - ++current_position; } result.pop_back(); // Remove the terminating quote character. diff --git a/boot/lexer.ll b/boot/lexer.ll index 691034a..c6d3b17 100644 --- a/boot/lexer.ll +++ b/boot/lexer.ll @@ -351,22 +351,28 @@ to { return yy::parser::make_FLOAT(result, this->location); } } -'([[:print:]]{-}['\\]|[[:blank:]]|\n|\r|\\.|\\\n)+' { - std::optional result = escape_string(yytext); - if (!result.has_value() || result.value().size() != 1) +'([^'\\\n]|\\.|\\\n)+' { + std::optional result = parse_byte_literal(yytext); + if (!result.has_value()) + { + REJECT; + } + return yy::parser::make_WORD8(std::make_pair(result.value(), integer_sign::_unsigned), this->location); +} +`([^'\\\n]|\\.|\\\n)+` { + std::optional result = parse_character_literal(yytext); + if (!result.has_value()) { REJECT; } - normalize_newlines(result.value()); return yy::parser::make_CHARACTER(result.value(), this->location); } -\"([[:print:]]{-}[\"\\]|[[:blank:]]|\n|\r|\\.|\\\n)*\" { - std::optional result = escape_string(yytext); +\"([^\"\\\n]|\\.|\\\n)*\" { + std::optional result = parse_string_literal(yytext); if (!result.has_value()) { REJECT; } - normalize_newlines(result.value()); return yy::parser::make_STRING(result.value(), this->location); } \( { diff --git a/boot/parser.yy b/boot/parser.yy index b48510b..1b3005e 100644 --- a/boot/parser.yy +++ b/boot/parser.yy @@ -83,7 +83,7 @@ along with GCC; see the file COPYING3. If not see %token > INTEGER16 WORD16 %token > INTEGER32 WORD32 %token FLOAT -%token CHARACTER +%token CHARACTER %token STRING %token BOOLEAN %token LEFT_PAREN "(" RIGHT_PAREN ")" LEFT_SQUARE "[" RIGHT_SQUARE "]" @@ -308,7 +308,7 @@ literal: } | CHARACTER { - $$ = new boot::literal(boot::make_position(@$), $1.at(0), boot::integer_sign::_unsigned); + $$ = new boot::literal(boot::make_position(@$), $1, boot::integer_sign::_unsigned); } | "nil" { diff --git a/include/elna/boot/driver.h b/include/elna/boot/driver.h index 9076e23..45e2e40 100644 --- a/include/elna/boot/driver.h +++ b/include/elna/boot/driver.h @@ -45,15 +45,31 @@ namespace elna::boot constexpr char escape_invalid_char = '\xff'; - char escape_char(char escape); /** - * Replaces both \r\n sequences and lone \r characters with \n. - * Modifies the original input. + * Parses a character literal and produces code point integer + * representation. * - * \param string Text to normalize. + * \param escape Text to parse. + * \return Code point. */ - void normalize_newlines(std::string& string); - std::optional escape_string(const char *escape); + std::optional parse_character_literal(const char *escape); + + /** + * Parses and unescapes a single byte character. Fails if the input inside + * single quotes contains more than a single character. Rejects unicode. + * + * \param escape Text to unescape. + * \return Unescaped character. + */ + std::optional parse_byte_literal(const char *escape); + + /** + * Parses and unescapes a string. + * + * \param escape Text to unescape. + * \return Unescaped string. + */ + std::optional parse_string_literal(const char *escape); template std::optional, integer_sign>> diff --git a/testsuite/runnable/unicode_escape_string.elna b/testsuite/runnable/unicode_escape_string.elna new file mode 100644 index 0000000..5905eda --- /dev/null +++ b/testsuite/runnable/unicode_escape_string.elna @@ -0,0 +1,4 @@ +begin + assert("caf\{U+E9}" = "café"); + assert("Hi \{U+1F600}!" = "Hi 😀!") +end. -- cgit v1.2.3