Support UTF-8 strings

1 files changed, 266 insertions, 38 deletions

diff --git a/boot/driver.cc b/boot/driver.cc
index 3607995..2bcbcb1 100644
--- a/boot/driver.cc
+++ b/boot/driver.cc
@@ -17,6 +17,8 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "elna/boot/driver.h"
 
+#include <algorithm>
+
 namespace elna::boot
 {
     source_position make_position(const yy::location& location)
@@ -39,7 +41,7 @@ namespace elna::boot
         return message;
     }
 
-    char escape_char(char escape)
+    static char unescape_character(char escape)
     {
         switch (escape)
         {
@@ -66,68 +68,294 @@ namespace elna::boot
         }
     }
 
-    void normalize_newlines(std::string& string)
+    static bool is_hex(char character)
+    {
+        return (character >= '0' && character <= '9')
+            || (character >= 'a' && character <= 'f')
+            || (character >= 'A' && character <= 'F');
+    }
+
+    constexpr unsigned char utf8_lead_tag_2byte = 0xC0;
+    constexpr unsigned char utf8_lead_tag_3byte = 0xE0;
+    constexpr unsigned char utf8_lead_tag_4byte = 0xF0;
+
+    constexpr int utf8_continuation_bits = 6;
+    constexpr uint32_t utf8_continuation_mask = 0x3F;
+    constexpr unsigned char utf8_continuation_tag = 0x80;
+
+    constexpr uint32_t utf8_surrogate_range_start = 0xD800;
+    constexpr uint32_t utf8_surrogate_range_end = 0xDFFF;
+    constexpr uint32_t utf8_max_unicode_codepoint = 0x10FFFF;
+
+    static std::optional<std::string> encode_utf8(uint32_t codepoint)
     {
+        constexpr uint32_t utf8_1byte_max = 0x80;
+        constexpr uint32_t utf8_2byte_max = 0x800;
+        constexpr uint32_t utf8_3byte_max = 0x10000;
+
         std::string result;
-        auto length = string.size();
 
-        for (std::size_t i = 0; i < length; ++i)
+        if ((codepoint >= utf8_surrogate_range_start && codepoint <= utf8_surrogate_range_end)
+                || codepoint > utf8_max_unicode_codepoint)
+        {
+            return std::nullopt;
+        }
+        else if (codepoint < utf8_1byte_max)
+        {
+            result.push_back(static_cast<char>(codepoint));
+        }
+        else if (codepoint < utf8_2byte_max)
+        {
+            result.push_back(static_cast<char>(utf8_continuation_tag | (codepoint & utf8_continuation_mask)));
+            result.push_back(static_cast<char>(utf8_lead_tag_2byte | (codepoint >> utf8_continuation_bits)));
+        }
+        else if (codepoint < utf8_3byte_max)
+        {
+            for (int i = 0; i < 2; ++i)
+            {
+                result.push_back(static_cast<char>(utf8_continuation_tag | (codepoint & utf8_continuation_mask)));
+                codepoint >>= utf8_continuation_bits;
+            }
+            result.push_back(static_cast<char>(utf8_lead_tag_3byte | codepoint));
+        }
+        else
+        {
+            for (int i = 0; i < 3; ++i)
+            {
+                result.push_back(static_cast<char>(utf8_continuation_tag | (codepoint & utf8_continuation_mask)));
+                codepoint >>= utf8_continuation_bits;
+            }
+            result.push_back(static_cast<char>(utf8_lead_tag_4byte | codepoint));
+        }
+        return std::optional(result);
+    }
+
+    static std::optional<std::pair<std::uint32_t, std::ptrdiff_t>> decode_utf8(const char *current_position)
+    {
+        constexpr unsigned char lead_tag_4byte_test = 0xF8;
+        constexpr uint32_t lead_payload_mask_2byte = 0x1F;
+        constexpr uint32_t lead_payload_mask_3byte = 0x0F;
+        constexpr uint32_t lead_payload_mask_4byte = 0x07;
+        constexpr unsigned char ascii_bit_mask = 0x80;
+
+        const unsigned char lead = static_cast<unsigned char>(current_position[0]);
+        std::ptrdiff_t extra;
+        uint32_t codepoint;
+
+        if ((lead & ascii_bit_mask) == 0x00)
+        {
+            extra = 0;
+            codepoint = lead;
+        }
+        else if ((lead & utf8_lead_tag_3byte) == utf8_lead_tag_2byte)
+        {
+            extra = 1;
+            codepoint = lead & lead_payload_mask_2byte;
+        }
+        else if ((lead & utf8_lead_tag_4byte) == utf8_lead_tag_3byte)
+        {
+            extra = 2;
+            codepoint = lead & lead_payload_mask_3byte;
+        }
+        else if ((lead & lead_tag_4byte_test) == utf8_lead_tag_4byte)
+        {
+            extra = 3;
+            codepoint = lead & lead_payload_mask_4byte;
+        }
+        else
+        {
+            return std::nullopt;
+        }
+        for (int i = 1; i <= extra; ++i)
         {
-            if (string[i] == '\r')
+            const unsigned char continuation_byte = static_cast<unsigned char>(current_position[i]);
+
+            if ((continuation_byte & utf8_lead_tag_2byte) != utf8_continuation_tag)
             {
-                if (i + 1 < length && string[i + 1] == '\n')
-                {
-                    ++i; // Skip \n in \r\n pairs.
-                }
-                result.push_back('\n');
+                return std::nullopt;
+            }
+            codepoint = (codepoint << utf8_continuation_bits) | (continuation_byte & utf8_continuation_mask);
+        }
+        return std::make_pair(codepoint, extra + 1);
+    }
+
+    static std::optional<std::pair<std::uint32_t, std::ptrdiff_t>> parse_codepoint_notation(const char *current_position)
+    {
+        if (current_position[0] != 'U' || current_position[1] != '+')
+        {
+            return std::nullopt;
+        }
+        current_position += 2;
+        std::ptrdiff_t consumed{ 2 };
+        const char *digits_start = current_position;
+
+        while (is_hex(*current_position))
+        {
+            ++current_position;
+        }
+        constexpr std::size_t max_digits = 8; // Unicode maximum is U+10FFFF.
+        consumed += current_position - digits_start;
+
+        if (current_position == digits_start || std::cmp_greater(consumed, max_digits))
+        {
+            return std::nullopt;
+        }
+        const std::string hex_digits(digits_start, current_position);
+        const std::uint32_t codepoint = static_cast<std::uint32_t>(std::stoul(hex_digits, nullptr, 16));
+
+        if ((codepoint >= utf8_surrogate_range_start && codepoint <= utf8_surrogate_range_end)
+                || codepoint > utf8_max_unicode_codepoint)
+        {
+            return std::nullopt;
+        }
+        return std::make_pair(codepoint, consumed);
+    }
+
+    static std::ptrdiff_t decode_one_character(const char *current_position, std::string& output)
+    {
+        if (current_position[0] == '\\' && current_position[1] == 'x')
+        {
+            current_position += 2;
+
+            if (!is_hex(current_position[0]) || !is_hex(current_position[1]))
+            {
+                return -1;
+            }
+            const std::string hex_digits(current_position, 2);
+            const char character = static_cast<char>(std::stoi(hex_digits, nullptr, 16));
+
+            output.push_back(character);
+
+            return 4;
+        }
+        else if (current_position[0] == '\\' && current_position[1] == '{')
+        {
+            current_position += 2;
+            auto unescaped = parse_codepoint_notation(current_position);
+
+            if (!unescaped.has_value())
+            {
+                return -1;
+            }
+            const auto [codepoint, consumed] = unescaped.value();
+
+            if (current_position[consumed] != '}')
+            {
+                return -1;
+            }
+            if (auto encoded = encode_utf8(codepoint))
+            {
+                std::ranges::reverse_copy(encoded.value(), std::back_inserter(output));
+
+                return consumed + 3;
             }
             else
             {
-                result.push_back(string[i]);
+                return -1;
+            }
+        }
+        else if (*current_position == '\\')
+        {
+            ++current_position;
+
+            const char escape = unescape_character(*current_position);
+            if (escape == escape_invalid_char)
+            {
+                return -1;
             }
+            output.push_back(escape);
+            return 2;
+        }
+        else
+        {
+            output.push_back(*current_position);
+            return 1;
         }
-        string = std::move(result);
     }
 
-    std::optional<std::string> escape_string(const char *escape)
+    std::optional<std::uint32_t> parse_character_literal(const char *escape)
     {
-        std::string result;
         const char *current_position = escape + 1;
+        std::uint32_t result{ 0 };
 
-        while (*current_position != '\0')
+        if (current_position[0] == '\\' && current_position[1] == '{')
         {
-            if (*current_position == '\\' && (*(current_position + 1) == 'x' || *(current_position + 1) == 'X'))
+            current_position += 2;
+            auto unescaped = parse_codepoint_notation(current_position);
+
+            if (!unescaped.has_value())
             {
-                current_position += 2;
+                return std::nullopt;
+            }
+            const auto [codepoint, consumed] = unescaped.value();
 
-                std::size_t processed;
-                char const character = static_cast<char>(std::stoi(current_position, &processed, 16));
-                if (processed == 0)
-                {
-                    return std::nullopt;
-                }
-                else
-                {
-                    current_position += processed - 1;
-                    result.push_back(character);
-                }
+            if (current_position[consumed] != '}')
+            {
+                return std::nullopt;
             }
-            else if (*current_position == '\\')
+            result = codepoint;
+            current_position += consumed + 1;
+        }
+        else if (current_position[0] == '\\')
+        {
+            char const escape = unescape_character(current_position[1]);
+
+            if (escape == escape_invalid_char)
             {
-                ++current_position;
+                return std::nullopt;
+            }
+            current_position += 2;
+            result = static_cast<std::uint32_t>(static_cast<unsigned char>(escape));
+        }
+        else
+        {
+            if (auto decoded = decode_utf8(current_position))
+            {
+                result = decoded->first;
+                current_position += decoded->second;
+            }
+            else
+            {
+                return std::nullopt;
+            }
+        }
+        return *current_position == '`' ? std::optional(result) : std::nullopt;
+    }
 
-                char const escape = escape_char(*current_position);
-                if (escape == escape_invalid_char)
-                {
-                    return std::nullopt;
-                }
-                result.push_back(escape);
+    std::optional<std::uint8_t> parse_byte_literal(const char *escape)
+    {
+        std::string result;
+        const char *current_position = escape + 1;
+        const ptrdiff_t consumed = decode_one_character(current_position, result);
+
+        // Unicode consumes more than 4 characters and is not supported here.
+        if (consumed == -1 || consumed > 4 || current_position[consumed] != '\'')
+        {
+            return std::nullopt;
+        }
+        else
+        {
+            return static_cast<std::uint8_t>(result.at(0));
+        }
+    }
+
+    std::optional<std::string> parse_string_literal(const char *escape)
+    {
+        std::string result;
+        const char *current_position = escape + 1;
+
+        while (*current_position != '\0')
+        {
+            const ptrdiff_t consumed = decode_one_character(current_position, result);
+
+            if (consumed == -1)
+            {
+                return std::nullopt;
             }
             else
             {
-                result.push_back(*current_position);
+                current_position += consumed;
             }
-            ++current_position;
         }
         result.pop_back(); // Remove the terminating quote character.