aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--boot/driver.cc304
-rw-r--r--boot/lexer.ll20
-rw-r--r--boot/parser.yy4
-rw-r--r--include/elna/boot/driver.h28
-rw-r--r--testsuite/runnable/unicode_escape_string.elna4
5 files changed, 307 insertions, 53 deletions
diff --git a/boot/driver.cc b/boot/driver.cc
index 3607995..2bcbcb1 100644
--- a/boot/driver.cc
+++ b/boot/driver.cc
@@ -17,6 +17,8 @@ along with GCC; see the file COPYING3. If not see
#include "elna/boot/driver.h"
+#include <algorithm>
+
namespace elna::boot
{
source_position make_position(const yy::location& location)
@@ -39,7 +41,7 @@ namespace elna::boot
return message;
}
- char escape_char(char escape)
+ static char unescape_character(char escape)
{
switch (escape)
{
@@ -66,68 +68,294 @@ namespace elna::boot
}
}
- void normalize_newlines(std::string& string)
+ static bool is_hex(char character)
+ {
+ return (character >= '0' && character <= '9')
+ || (character >= 'a' && character <= 'f')
+ || (character >= 'A' && character <= 'F');
+ }
+
+ constexpr unsigned char utf8_lead_tag_2byte = 0xC0;
+ constexpr unsigned char utf8_lead_tag_3byte = 0xE0;
+ constexpr unsigned char utf8_lead_tag_4byte = 0xF0;
+
+ constexpr int utf8_continuation_bits = 6;
+ constexpr uint32_t utf8_continuation_mask = 0x3F;
+ constexpr unsigned char utf8_continuation_tag = 0x80;
+
+ constexpr uint32_t utf8_surrogate_range_start = 0xD800;
+ constexpr uint32_t utf8_surrogate_range_end = 0xDFFF;
+ constexpr uint32_t utf8_max_unicode_codepoint = 0x10FFFF;
+
+ static std::optional<std::string> encode_utf8(uint32_t codepoint)
{
+ constexpr uint32_t utf8_1byte_max = 0x80;
+ constexpr uint32_t utf8_2byte_max = 0x800;
+ constexpr uint32_t utf8_3byte_max = 0x10000;
+
std::string result;
- auto length = string.size();
- for (std::size_t i = 0; i < length; ++i)
+ if ((codepoint >= utf8_surrogate_range_start && codepoint <= utf8_surrogate_range_end)
+ || codepoint > utf8_max_unicode_codepoint)
+ {
+ return std::nullopt;
+ }
+ else if (codepoint < utf8_1byte_max)
+ {
+ result.push_back(static_cast<char>(codepoint));
+ }
+ else if (codepoint < utf8_2byte_max)
+ {
+ result.push_back(static_cast<char>(utf8_continuation_tag | (codepoint & utf8_continuation_mask)));
+ result.push_back(static_cast<char>(utf8_lead_tag_2byte | (codepoint >> utf8_continuation_bits)));
+ }
+ else if (codepoint < utf8_3byte_max)
+ {
+ for (int i = 0; i < 2; ++i)
+ {
+ result.push_back(static_cast<char>(utf8_continuation_tag | (codepoint & utf8_continuation_mask)));
+ codepoint >>= utf8_continuation_bits;
+ }
+ result.push_back(static_cast<char>(utf8_lead_tag_3byte | codepoint));
+ }
+ else
+ {
+ for (int i = 0; i < 3; ++i)
+ {
+ result.push_back(static_cast<char>(utf8_continuation_tag | (codepoint & utf8_continuation_mask)));
+ codepoint >>= utf8_continuation_bits;
+ }
+ result.push_back(static_cast<char>(utf8_lead_tag_4byte | codepoint));
+ }
+ return std::optional(result);
+ }
+
+ static std::optional<std::pair<std::uint32_t, std::ptrdiff_t>> decode_utf8(const char *current_position)
+ {
+ constexpr unsigned char lead_tag_4byte_test = 0xF8;
+ constexpr uint32_t lead_payload_mask_2byte = 0x1F;
+ constexpr uint32_t lead_payload_mask_3byte = 0x0F;
+ constexpr uint32_t lead_payload_mask_4byte = 0x07;
+ constexpr unsigned char ascii_bit_mask = 0x80;
+
+ const unsigned char lead = static_cast<unsigned char>(current_position[0]);
+ std::ptrdiff_t extra;
+ uint32_t codepoint;
+
+ if ((lead & ascii_bit_mask) == 0x00)
+ {
+ extra = 0;
+ codepoint = lead;
+ }
+ else if ((lead & utf8_lead_tag_3byte) == utf8_lead_tag_2byte)
+ {
+ extra = 1;
+ codepoint = lead & lead_payload_mask_2byte;
+ }
+ else if ((lead & utf8_lead_tag_4byte) == utf8_lead_tag_3byte)
+ {
+ extra = 2;
+ codepoint = lead & lead_payload_mask_3byte;
+ }
+ else if ((lead & lead_tag_4byte_test) == utf8_lead_tag_4byte)
+ {
+ extra = 3;
+ codepoint = lead & lead_payload_mask_4byte;
+ }
+ else
+ {
+ return std::nullopt;
+ }
+ for (int i = 1; i <= extra; ++i)
{
- if (string[i] == '\r')
+ const unsigned char continuation_byte = static_cast<unsigned char>(current_position[i]);
+
+ if ((continuation_byte & utf8_lead_tag_2byte) != utf8_continuation_tag)
{
- if (i + 1 < length && string[i + 1] == '\n')
- {
- ++i; // Skip \n in \r\n pairs.
- }
- result.push_back('\n');
+ return std::nullopt;
+ }
+ codepoint = (codepoint << utf8_continuation_bits) | (continuation_byte & utf8_continuation_mask);
+ }
+ return std::make_pair(codepoint, extra + 1);
+ }
+
+ static std::optional<std::pair<std::uint32_t, std::ptrdiff_t>> parse_codepoint_notation(const char *current_position)
+ {
+ if (current_position[0] != 'U' || current_position[1] != '+')
+ {
+ return std::nullopt;
+ }
+ current_position += 2;
+ std::ptrdiff_t consumed{ 2 };
+ const char *digits_start = current_position;
+
+ while (is_hex(*current_position))
+ {
+ ++current_position;
+ }
+ constexpr std::size_t max_digits = 8; // Unicode maximum is U+10FFFF.
+ consumed += current_position - digits_start;
+
+ if (current_position == digits_start || std::cmp_greater(consumed, max_digits))
+ {
+ return std::nullopt;
+ }
+ const std::string hex_digits(digits_start, current_position);
+ const std::uint32_t codepoint = static_cast<std::uint32_t>(std::stoul(hex_digits, nullptr, 16));
+
+ if ((codepoint >= utf8_surrogate_range_start && codepoint <= utf8_surrogate_range_end)
+ || codepoint > utf8_max_unicode_codepoint)
+ {
+ return std::nullopt;
+ }
+ return std::make_pair(codepoint, consumed);
+ }
+
+ static std::ptrdiff_t decode_one_character(const char *current_position, std::string& output)
+ {
+ if (current_position[0] == '\\' && current_position[1] == 'x')
+ {
+ current_position += 2;
+
+ if (!is_hex(current_position[0]) || !is_hex(current_position[1]))
+ {
+ return -1;
+ }
+ const std::string hex_digits(current_position, 2);
+ const char character = static_cast<char>(std::stoi(hex_digits, nullptr, 16));
+
+ output.push_back(character);
+
+ return 4;
+ }
+ else if (current_position[0] == '\\' && current_position[1] == '{')
+ {
+ current_position += 2;
+ auto unescaped = parse_codepoint_notation(current_position);
+
+ if (!unescaped.has_value())
+ {
+ return -1;
+ }
+ const auto [codepoint, consumed] = unescaped.value();
+
+ if (current_position[consumed] != '}')
+ {
+ return -1;
+ }
+ if (auto encoded = encode_utf8(codepoint))
+ {
+ std::ranges::reverse_copy(encoded.value(), std::back_inserter(output));
+
+ return consumed + 3;
}
else
{
- result.push_back(string[i]);
+ return -1;
+ }
+ }
+ else if (*current_position == '\\')
+ {
+ ++current_position;
+
+ const char escape = unescape_character(*current_position);
+ if (escape == escape_invalid_char)
+ {
+ return -1;
}
+ output.push_back(escape);
+ return 2;
+ }
+ else
+ {
+ output.push_back(*current_position);
+ return 1;
}
- string = std::move(result);
}
- std::optional<std::string> escape_string(const char *escape)
+ std::optional<std::uint32_t> parse_character_literal(const char *escape)
{
- std::string result;
const char *current_position = escape + 1;
+ std::uint32_t result{ 0 };
- while (*current_position != '\0')
+ if (current_position[0] == '\\' && current_position[1] == '{')
{
- if (*current_position == '\\' && (*(current_position + 1) == 'x' || *(current_position + 1) == 'X'))
+ current_position += 2;
+ auto unescaped = parse_codepoint_notation(current_position);
+
+ if (!unescaped.has_value())
{
- current_position += 2;
+ return std::nullopt;
+ }
+ const auto [codepoint, consumed] = unescaped.value();
- std::size_t processed;
- char const character = static_cast<char>(std::stoi(current_position, &processed, 16));
- if (processed == 0)
- {
- return std::nullopt;
- }
- else
- {
- current_position += processed - 1;
- result.push_back(character);
- }
+ if (current_position[consumed] != '}')
+ {
+ return std::nullopt;
}
- else if (*current_position == '\\')
+ result = codepoint;
+ current_position += consumed + 1;
+ }
+ else if (current_position[0] == '\\')
+ {
+ char const escape = unescape_character(current_position[1]);
+
+ if (escape == escape_invalid_char)
{
- ++current_position;
+ return std::nullopt;
+ }
+ current_position += 2;
+ result = static_cast<std::uint32_t>(static_cast<unsigned char>(escape));
+ }
+ else
+ {
+ if (auto decoded = decode_utf8(current_position))
+ {
+ result = decoded->first;
+ current_position += decoded->second;
+ }
+ else
+ {
+ return std::nullopt;
+ }
+ }
+ return *current_position == '`' ? std::optional(result) : std::nullopt;
+ }
- char const escape = escape_char(*current_position);
- if (escape == escape_invalid_char)
- {
- return std::nullopt;
- }
- result.push_back(escape);
+ std::optional<std::uint8_t> parse_byte_literal(const char *escape)
+ {
+ std::string result;
+ const char *current_position = escape + 1;
+ const ptrdiff_t consumed = decode_one_character(current_position, result);
+
+ // Unicode consumes more than 4 characters and is not supported here.
+ if (consumed == -1 || consumed > 4 || current_position[consumed] != '\'')
+ {
+ return std::nullopt;
+ }
+ else
+ {
+ return static_cast<std::uint8_t>(result.at(0));
+ }
+ }
+
+ std::optional<std::string> parse_string_literal(const char *escape)
+ {
+ std::string result;
+ const char *current_position = escape + 1;
+
+ while (*current_position != '\0')
+ {
+ const ptrdiff_t consumed = decode_one_character(current_position, result);
+
+ if (consumed == -1)
+ {
+ return std::nullopt;
}
else
{
- result.push_back(*current_position);
+ current_position += consumed;
}
- ++current_position;
}
result.pop_back(); // Remove the terminating quote character.
diff --git a/boot/lexer.ll b/boot/lexer.ll
index 691034a..c6d3b17 100644
--- a/boot/lexer.ll
+++ b/boot/lexer.ll
@@ -351,22 +351,28 @@ to {
return yy::parser::make_FLOAT(result, this->location);
}
}
-'([[:print:]]{-}['\\]|[[:blank:]]|\n|\r|\\.|\\\n)+' {
- std::optional<std::string> result = escape_string(yytext);
- if (!result.has_value() || result.value().size() != 1)
+'([^'\\\n]|\\.|\\\n)+' {
+ std::optional<std::uint8_t> result = parse_byte_literal(yytext);
+ if (!result.has_value())
+ {
+ REJECT;
+ }
+ return yy::parser::make_WORD8(std::make_pair(result.value(), integer_sign::_unsigned), this->location);
+}
+`([^'\\\n]|\\.|\\\n)+` {
+ std::optional<std::uint32_t> result = parse_character_literal(yytext);
+ if (!result.has_value())
{
REJECT;
}
- normalize_newlines(result.value());
return yy::parser::make_CHARACTER(result.value(), this->location);
}
-\"([[:print:]]{-}[\"\\]|[[:blank:]]|\n|\r|\\.|\\\n)*\" {
- std::optional<std::string> result = escape_string(yytext);
+\"([^\"\\\n]|\\.|\\\n)*\" {
+ std::optional<std::string> result = parse_string_literal(yytext);
if (!result.has_value())
{
REJECT;
}
- normalize_newlines(result.value());
return yy::parser::make_STRING(result.value(), this->location);
}
\( {
diff --git a/boot/parser.yy b/boot/parser.yy
index b48510b..1b3005e 100644
--- a/boot/parser.yy
+++ b/boot/parser.yy
@@ -83,7 +83,7 @@ along with GCC; see the file COPYING3. If not see
%token <std::pair<std::uint16_t, elna::boot::integer_sign>> INTEGER16 WORD16
%token <std::pair<std::uint32_t, elna::boot::integer_sign>> INTEGER32 WORD32
%token <double> FLOAT
-%token <std::string> CHARACTER
+%token <std::uint32_t> CHARACTER
%token <std::string> STRING
%token <bool> BOOLEAN
%token LEFT_PAREN "(" RIGHT_PAREN ")" LEFT_SQUARE "[" RIGHT_SQUARE "]"
@@ -308,7 +308,7 @@ literal:
}
| CHARACTER
{
- $$ = new boot::literal<unsigned char>(boot::make_position(@$), $1.at(0), boot::integer_sign::_unsigned);
+ $$ = new boot::literal<unsigned char>(boot::make_position(@$), $1, boot::integer_sign::_unsigned);
}
| "nil"
{
diff --git a/include/elna/boot/driver.h b/include/elna/boot/driver.h
index 9076e23..45e2e40 100644
--- a/include/elna/boot/driver.h
+++ b/include/elna/boot/driver.h
@@ -45,15 +45,31 @@ namespace elna::boot
constexpr char escape_invalid_char = '\xff';
- char escape_char(char escape);
/**
- * Replaces both \r\n sequences and lone \r characters with \n.
- * Modifies the original input.
+ * Parses a character literal and produces code point integer
+ * representation.
*
- * \param string Text to normalize.
+ * \param escape Text to parse.
+ * \return Code point.
*/
- void normalize_newlines(std::string& string);
- std::optional<std::string> escape_string(const char *escape);
+ std::optional<std::uint32_t> parse_character_literal(const char *escape);
+
+ /**
+ * Parses and unescapes a single byte character. Fails if the input inside
+ * single quotes contains more than a single character. Rejects unicode.
+ *
+ * \param escape Text to unescape.
+ * \return Unescaped character.
+ */
+ std::optional<std::uint8_t> parse_byte_literal(const char *escape);
+
+ /**
+ * Parses and unescapes a string.
+ *
+ * \param escape Text to unescape.
+ * \return Unescaped string.
+ */
+ std::optional<std::string> parse_string_literal(const char *escape);
template<typename T>
std::optional<std::pair<std::make_unsigned_t<T>, integer_sign>>
diff --git a/testsuite/runnable/unicode_escape_string.elna b/testsuite/runnable/unicode_escape_string.elna
new file mode 100644
index 0000000..5905eda
--- /dev/null
+++ b/testsuite/runnable/unicode_escape_string.elna
@@ -0,0 +1,4 @@
+begin
+ assert("caf\{U+E9}" = "café");
+ assert("Hi \{U+1F600}!" = "Hi 😀!")
+end.
generated by cgit v1.2.3 (git 2.26.2) at 2026-09-01 06:50:35 +0000