belka/elna
1
0
Fork 0
Code Issues 5 Pull Requests Releases Activity

Tokenize the input with flex

Browse Source
This commit is contained in:
Eugen Wissner
2024-04-06 16:10:07 +02:00
parent 0cad759415
commit 0876e25f90
11 changed files with 274 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
include/elna/cli/cl.hpp
View File

@ -6,15 +6,6 @@
namespace elna::cli
{
/**
* Reads an input file and returns its contents.
*
* \param source Input file.
*
* \return File contents.
*/
std::string read_source(const char *source);
/**
* Formats and prints the given error.
*

40
include/elna/source/lexer.hpp
View File

@ -9,40 +9,6 @@
namespace elna::source
{
/**
* Range over the source text that keeps track of the current position.
*/
class text_iterator
{
std::string::const_iterator m_buffer;
elna::source::position m_position;
text_iterator(std::string::const_iterator buffer,
const elna::source::position start_position = elna::source::position());
public:
using iterator_category = std::forward_iterator_tag;
using difference_type = ptrdiff_t;
using value_type = char;
using pointer = const value_type *;
using reference = const value_type&;
const elna::source::position& position() const noexcept;
reference operator*() const noexcept;
pointer operator->() const noexcept;
text_iterator& operator++();
text_iterator& operator++(int);
bool operator==(const text_iterator& that) const noexcept;
bool operator!=(const text_iterator& that) const noexcept;
text_iterator operator+(std::size_t step);
friend std::pair<text_iterator, text_iterator> text_iterators(const std::string& buffer);
};
std::pair<text_iterator, text_iterator>
text_iterators(const std::string &buffer);
/**
* Union type representing a single token.
*/
@ -53,6 +19,7 @@ namespace elna::source
*/
enum class type : std::uint16_t
{
dot,
number,
boolean,
term_operator,
@ -63,7 +30,6 @@ namespace elna::source
semicolon,
left_paren,
right_paren,
dot,
comma,
factor_operator,
eof,
@ -97,6 +63,7 @@ namespace elna::source
token(type of, elna::source::position position);
token(type of, std::int32_t value, const elna::source::position position);
token(type of, const std::string& value, const elna::source::position position);
token(type of, value&& value, const elna::source::position position);
token(const token& that);
token(token&& that);
~token();
@ -236,9 +203,8 @@ namespace elna::source
/**
* Splits the source text into tokens.
*
* \param buffer Source text.
* \param path Source file location.
* \return Tokens or error.
*/
elna::source::result<lexer> lex(const std::string& buffer, const std::filesystem::path& path);
elna::source::result<lexer> tokenize(const std::filesystem::path& path);
}

43
include/elna/source/parser.hpp
View File

@ -89,35 +89,44 @@ namespace elna::source
};
/**
* Variable declaration.
*/
class declaration : public node
{
std::string m_identifier;
std::string m_type;
public:
declaration(const std::string& identifier, const std::string& type);
virtual void accept(parser_visitor *visitor) override;
std::string& identifier() noexcept;
std::string& type() noexcept;
};
/**
* Constant definition.
* Symbol definition.
*/
class definition : public node
{
std::string m_identifier;
protected:
/**
* Constructs a definition identified by some name.
*
* \param identifier Definition name.
*/
definition(const std::string& identifier);
public:
/**
* \return Definition name.
*/
std::string& identifier() noexcept;
};
/**
* Variable declaration.
*/
class declaration : public definition
{
std::string m_type;
public:
declaration(const std::string& identifier, const std::string& type);
virtual void accept(parser_visitor *visitor) override;
std::string& type() noexcept;
};
/**
* Constant definition.
*/
class constant_definition : public definition
{
std::unique_ptr<integer_literal> m_body;