Run tests and check expectations in 2 loops

TODO

@@ -12,5 +12,3 @@
 - Syscalls.
 - Error message with an empty file wrongly says that a ")" is expected.
 - Support any expressions for constants.
-- Built-in types are added in the symbol table constructor. It would be better to
-  add them outside, before the type analysis begins.

cli/cl.cpp

@@ -2,6 +2,7 @@
 #include "elna/backend/target.hpp"
 #include "elna/source/semantic.hpp"
 #include "elna/source/optimizer.hpp"
+#include "elna/source/types.hpp"
 #include <iostream>
 
 namespace elna::cli
@@ -13,6 +14,26 @@ namespace elna::cli
             << ": " << compile_error->what() << std::endl;
     }
 
+    static std::shared_ptr<source::symbol_table> add_builtin_symbols()
+    {
+        source::symbol_table result;
+
+        auto boolean_info = std::make_shared<source::type_info>(source::boolean_type);
+        auto int_info = std::make_shared<source::type_info>(source::int_type);
+        result.enter("Boolean", boolean_info);
+        result.enter("Int", int_info);
+
+        auto writei = std::make_shared<source::intrinsic_info>();
+        writei->parameter_infos.emplace_back(int_info->type());
+        result.enter("writei", writei);
+
+        auto writeb = std::make_shared<source::intrinsic_info>();
+        writeb->parameter_infos.emplace_back(boolean_info->type());
+        result.enter("writeb", writeb);
+
+        return std::make_shared<source::symbol_table>(std::move(result));
+    }
+
     int compile(const std::filesystem::path& in_file, const std::filesystem::path& out_file)
     {
         try
@@ -31,7 +52,7 @@ namespace elna::cli
                 print_errors(parser.errors().cbegin(), parser.errors().cend());
                 return 2;
             }
-            auto global_scope = std::make_shared<source::symbol_table>();
+            auto global_scope = add_builtin_symbols();
             source::name_analysis_visitor(global_scope).visit(ast.get());
             source::type_analysis_visitor().visit(ast.get());
             source::allocator_visitor(global_scope).visit(ast.get());

include/elna/source/parser.hpp

@@ -204,6 +204,9 @@ namespace elna::source
         std::string& identifier() noexcept;
     };
 
+    /**
+     * Expression defining a composed type like pointer or an array.
+     */
     class type_expression : public node
     {
         std::string m_base;

include/elna/tester.hpp

@@ -21,13 +21,6 @@ namespace elna
         expectation_not_found
     };
 
-    struct test_result final
-    {
-        test_status status{ test_status::successful };
-        int code{ 0 };
-        std::string output;
-    };
-
     class test_results final
     {
         std::uint32_t m_total{ 0 };
@@ -46,10 +39,10 @@ namespace elna
 
     boost::process::v2::process_stdio get_output_streams(const std::uint8_t stream_number,
             boost::asio::readable_pipe& read_pipe);
-    test_result run_for_output(boost::asio::io_context& context, const std::uint8_t stream_number,
+    int run_for_output(boost::asio::io_context& context, const std::uint8_t stream_number,
             const std::filesystem::path& binary, std::initializer_list<boost::string_view> arguments);
     std::string find_object(const std::vector<std::filesystem::path>& environment, const std::string& object);
-    test_result build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry);
+    test_status build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry);
-    test_result run_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry);
+    void run_test(boost::asio::io_context& context, const std::filesystem::path& test_entry);
-    void print_result(const std::filesystem::directory_entry& test_entry, const test_result& result);
+    void print_result(const std::filesystem::path& test_entry, const test_status& result);
 }

source/symbol_table.cpp

@@ -6,21 +6,6 @@ namespace elna::source
     symbol_table::symbol_table(std::shared_ptr<symbol_table> scope)
         : outer_scope(scope)
     {
-        if (scope == nullptr)
-        {
-            auto boolean_info = std::make_shared<type_info>(boolean_type);
-            auto int_info = std::make_shared<type_info>(int_type);
-            enter("Boolean", boolean_info);
-            enter("Int", int_info);
-
-            auto writei = std::make_shared<intrinsic_info>();
-            writei->parameter_infos.emplace_back(int_info->type());
-            enter("writei", writei);
-
-            auto writeb = std::make_shared<intrinsic_info>();
-            writeb->parameter_infos.emplace_back(boolean_info->type());
-            enter("writeb", writeb);
-        }
     }
 
     std::shared_ptr<info> symbol_table::lookup(const std::string& name)

tests/tester.cpp

@@ -1,6 +1,7 @@
 #include "elna/tester.hpp"
 
 #include <iostream>
+#include <fstream>
 #include <algorithm>
 #include <boost/process/env.hpp>
 
@@ -45,6 +46,11 @@ namespace elna
         return in_build_directory() / path;
     }
 
+    static std::filesystem::path in_actual_directory()
+    {
+        return "build/tests";
+    }
+
     boost::process::v2::process_stdio get_output_streams(const std::uint8_t stream_number,
             boost::asio::readable_pipe& read_pipe)
     {
@@ -59,19 +65,18 @@ namespace elna
         return boost::process::v2::process_stdio{ nullptr, read_pipe, read_pipe };
     }
 
-    test_result run_for_output(boost::asio::io_context& context, const std::uint8_t stream_number,
+    int run_for_output(boost::asio::io_context& context, const std::filesystem::path& log_path,
             const std::filesystem::path& binary, std::initializer_list<boost::string_view> arguments)
     {
         boost::asio::readable_pipe read_pipe{ context };
-        test_result result{};
         std::string output;
         boost::asio::dynamic_string_buffer buffer = boost::asio::dynamic_buffer(output);
         boost::system::error_code ec;
 
         boost::process::v2::process_stdio process_stdio;
-        boost::process::v2::process elna_child(context,
+        boost::process::v2::process elna_child(context, binary, arguments,
-                binary, arguments,
+                boost::process::v2::process_stdio{ nullptr, read_pipe, 1 });
-                get_output_streams(stream_number, read_pipe));
+        std::ofstream log_file{ log_path };
         do
         {
             std::size_t transferred = read_pipe.read_some(buffer.prepare(512), ec);
@@ -81,13 +86,12 @@ namespace elna
                 break;
             }
             buffer.commit(transferred);
-            result.output.append(boost::asio::buffer_cast<const char *>(buffer.data()), buffer.size());
+            log_file.write(boost::asio::buffer_cast<const char *>(buffer.data()), buffer.size());
             buffer.consume(transferred);
         }
         while (ec == boost::system::errc::success);
-        result.code = elna_child.wait();
 
-        return result;
+        return elna_child.wait();
     }
 
     std::string find_object(const std::vector<std::filesystem::path>& environment, const std::string& object)
@@ -106,26 +110,27 @@ namespace elna
         return object;
     }
 
-    test_result build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
+    test_status build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
     {
         const std::filesystem::path test_filename = test_entry.path().filename();
 
         std::filesystem::path test_binary = in_build_directory(test_filename);
         std::filesystem::path test_object = test_binary;
+        std::filesystem::path test_log = test_binary;
         test_binary.replace_extension();
         test_object.replace_extension(".o");
+        test_log.replace_extension(".log");
 
         std::filesystem::remove(test_binary);
         std::filesystem::remove(test_object);
 
-        test_result result = run_for_output(context, 2, "./build/bin/elna",
+        auto result_code = run_for_output(context, test_log, "./build/bin/elna",
                 { "-o", test_object.string(), test_entry.path().string() });
-
+        if (result_code != 0)
-        if (result.code != 0)
         {
-            result.status = test_status::compile_failed;
+            return test_status::compile_failed;
-            return result;
         }
+        test_status result = result_code == 0 ? test_status::successful : test_status::compile_failed;
         std::vector<std::filesystem::path> environment;
         std::vector<std::string> linker_arguments = { "-o", test_binary.string() };
 
@@ -141,64 +146,40 @@ namespace elna
                 { "--start-group", "-lgcc", "-lc", "-lgloss", "--end-group" });
         linker_arguments.push_back(find_object(environment, "crtend.o"));
 
-        boost::process::v2::execute(boost::process::v2::process(context,
+        boost::process::v2::execute(boost::process::v2::process(context, boost::process::search_path("ld"),
-                    boost::process::search_path("ld"),
                     linker_arguments
         ));
-        return test_result{};
+        return result;
     }
 
-    static test_result check_expectation(const std::filesystem::directory_entry& test_entry, const test_result& run)
+    static test_status check_expectation(const std::filesystem::path& expectation_path,
+            const std::filesystem::path& actual_directory)
     {
-        std::filesystem::path test_filename = test_entry.path().filename();
+        std::filesystem::path log_filename = expectation_path.filename();
-        test_filename.replace_extension(".txt");
+        log_filename.replace_extension(".log");
 
-        const std::filesystem::path expectation_path =
+        std::filesystem::path test_log = actual_directory / log_filename;
-            test_entry.path().parent_path() / "expectations" / test_filename;
-        const std::filesystem::path failures_path =
-            test_entry.path().parent_path() / "failures" / test_filename;
 
-        std::string expected_result_path;
+        test_log.replace_extension(".log");
-        if (std::filesystem::exists(expectation_path))
-        {
-            expected_result_path = expectation_path.string();
-        }
-        else if (std::filesystem::exists(failures_path))
-        {
-            expected_result_path = failures_path.string();
-        }
-        else
-        {
-            return test_result{ test_status::expectation_not_found, run.code, run.output };
-        }
-        boost::process::opstream pipe_stream;
         boost::process::child diff(
                 boost::process::search_path("diff"), "-Nur", "--color",
-                expected_result_path, "-",
+                expectation_path, test_log
-                boost::process::std_in < pipe_stream
         );
-
-        pipe_stream << run.output;
-        pipe_stream.flush();
-        pipe_stream.pipe().close();
         diff.wait();
 
         if (diff.exit_code() == 0)
         {
-            return test_result{ test_status::successful, run.code, run.output };
+            return test_status::successful;
         }
-        return test_result{ test_status::expectation_failed, run.code, run.output };
+        return test_status::expectation_failed;
     }
 
-    test_result run_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
+    void run_test(boost::asio::io_context& context, const std::filesystem::path& test_binary)
     {
-        const std::filesystem::path test_filename = test_entry.path().filename();
+        std::filesystem::path test_log = test_binary;
+        test_log.replace_extension(".log");
 
-        std::filesystem::path test_binary = in_build_directory(test_filename);
+        run_for_output(context, test_log, boost::process::search_path("qemu-riscv32"),
-        test_binary.replace_extension();
-
-        return run_for_output(context, 1,
-                boost::process::search_path("qemu-riscv32"),
                 { test_binary.string() });
     }
 
@@ -213,33 +194,51 @@ namespace elna
             {
                 continue;
             }
-            test_result result;
+            test_status result;
 
             std::cout << "Running " << test_entry << std::endl;
             try
             {
-                result = build_test(context, test_entry);
+                build_test(context, test_entry);
-                if (result.status == test_status::successful)
-                {
-                    result = run_test(context, test_entry);
-                }
-                result = check_expectation(test_entry, result);
             }
             catch (const boost::process::process_error& exception)
             {
-                result.status = test_status::build_failed;
+                test_status::build_failed;
-                result.output = exception.what();
+                std::cout << exception.what() << std::endl;
-                std::cout << result.output << std::endl;
             }
+        }
+        for (const auto& executable_test : std::filesystem::directory_iterator(in_build_directory()))
+        {
+            if (executable_test.path().has_extension())
+            {
+                continue;
+            }
+            run_test(context, executable_test.path());
+        }
+        for (const auto& expectation_entry : std::filesystem::directory_iterator(test_directory / "failures"))
+        {
+            auto test_entry = test_directory / expectation_entry.path().filename();
+            test_entry.replace_extension(".eln");
+            auto result = check_expectation(expectation_entry.path(), in_build_directory());
+
             print_result(test_entry, result);
-            results.add_exit_code(result.status);
+            results.add_exit_code(result);
+        }
+        for (const auto& expectation_entry : std::filesystem::directory_iterator(test_directory / "expectations"))
+        {
+            auto test_entry = test_directory / expectation_entry.path().filename();
+            test_entry.replace_extension(".eln");
+            auto result = check_expectation(expectation_entry.path(), in_actual_directory());
+
+            print_result(test_entry, result);
+            results.add_exit_code(result);
         }
         return results;
     }
 
-    void print_result(const std::filesystem::directory_entry& test_entry, const test_result& result)
+    void print_result(const std::filesystem::path& test_entry, const test_status& result)
     {
-        if (result.status != test_status::successful)
+        if (result != test_status::successful)
         {
             std::cout << test_entry << " failed." << std::endl;
         }
@@ -249,6 +248,7 @@ namespace elna
 int main()
 {
     std::filesystem::create_directory(elna::in_build_directory());
+    std::filesystem::create_directory(elna::in_actual_directory());
 
     std::cout << "Run all tests and check the results" << std::endl;
     std::filesystem::path test_directory{ "tests" };