elna/tests/tester.cpp

224 lines
7.4 KiB
C++
Raw Normal View History

2024-03-14 08:52:45 +01:00
#include "elna/tester.hpp"
2024-02-18 11:26:57 +01:00
2024-02-22 21:29:25 +01:00
#include <iostream>
2024-03-14 08:52:45 +01:00
2024-02-22 21:29:25 +01:00
namespace elna
2024-02-18 11:26:57 +01:00
{
2024-02-22 21:29:25 +01:00
std::uint32_t test_results::total() const noexcept
2024-02-18 11:26:57 +01:00
{
return m_total;
}
2024-02-22 21:29:25 +01:00
std::uint32_t test_results::passed() const noexcept
2024-02-18 11:26:57 +01:00
{
return m_passed;
}
2024-02-22 21:29:25 +01:00
std::uint32_t test_results::failed() const noexcept
2024-02-18 11:26:57 +01:00
{
return m_total - m_passed;
}
2024-02-22 21:29:25 +01:00
int test_results::exit_code() const noexcept
2024-02-18 11:26:57 +01:00
{
return m_total == m_passed ? EXIT_SUCCESS : EXIT_FAILURE;
}
2024-03-14 08:52:45 +01:00
void test_results::add_exit_code(const test_status status) noexcept
2024-02-18 11:26:57 +01:00
{
++m_total;
2024-03-14 08:52:45 +01:00
if (status == test_status::successful)
2024-02-18 11:26:57 +01:00
{
++m_passed;
}
}
2024-03-03 13:11:39 +01:00
static std::filesystem::path in_build_directory()
{
return "build/riscv";
}
2024-02-25 15:16:19 +01:00
static std::string in_build_directory(const std::filesystem::path& path)
{
2024-03-03 13:11:39 +01:00
return in_build_directory() / path;
2024-02-25 15:16:19 +01:00
}
2024-03-23 14:53:26 +01:00
test_result run_for_output(boost::asio::io_context& context, const std::filesystem::path& binary,
std::initializer_list<boost::string_view> arguments)
{
boost::asio::readable_pipe read_pipe{ context };
test_result result{};
std::string output;
boost::asio::dynamic_string_buffer buffer = boost::asio::dynamic_buffer(output);
boost::system::error_code ec;
boost::process::v2::process elna_child(context,
binary, arguments,
boost::process::v2::process_stdio{ nullptr, read_pipe, read_pipe });
do
{
std::size_t transferred = read_pipe.read_some(buffer.prepare(512), ec);
if (transferred == 0)
{
break;
}
buffer.commit(transferred);
result.output.append(boost::asio::buffer_cast<const char *>(buffer.data()), buffer.size());
buffer.consume(transferred);
}
while (ec == boost::system::errc::success);
result.code = elna_child.wait();
return result;
}
test_result build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
2024-02-25 15:16:19 +01:00
{
const std::filesystem::path test_filename = test_entry.path().filename();
std::filesystem::path test_binary = in_build_directory(test_filename);
std::filesystem::path test_object = test_binary;
test_binary.replace_extension();
test_object.replace_extension(".o");
std::filesystem::remove(test_binary);
std::filesystem::remove(test_object);
2024-03-23 14:53:26 +01:00
test_result result = run_for_output(context, "./build/bin/elna",
{ "-o", test_object.string(), test_entry.path().string() });
if (result.code != 0)
2024-02-25 15:16:19 +01:00
{
2024-03-23 14:53:26 +01:00
result.status = test_status::compile_failed;
return result;
2024-02-25 15:16:19 +01:00
}
2024-03-23 14:53:26 +01:00
boost::process::v2::execute(boost::process::v2::process(
context, "/opt/riscv/bin/riscv32-unknown-elf-ld",
{
"-o", test_binary.string(),
"-L/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/",
"-L/opt/riscv/riscv32-unknown-elf/lib",
"/opt/riscv/riscv32-unknown-elf/lib/crt0.o",
"/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtbegin.o",
test_object.string(),
"--start-group", "-lgcc", "-lc", "-lgloss", "--end-group",
"/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtend.o"
}
));
2024-03-14 08:52:45 +01:00
return test_result{};
2024-02-25 15:16:19 +01:00
}
2024-03-14 08:52:45 +01:00
static test_result check_expectation(const std::filesystem::directory_entry& test_entry, const test_result& run)
2024-02-22 21:29:25 +01:00
{
2024-03-23 14:53:26 +01:00
std::filesystem::path test_filename = test_entry.path().filename();
test_filename.replace_extension(".txt");
2024-02-22 21:29:25 +01:00
2024-03-23 14:53:26 +01:00
const std::filesystem::path expectation_path =
test_entry.path().parent_path() / "expectations" / test_filename;
const std::filesystem::path failures_path =
test_entry.path().parent_path() / "failures" / test_filename;
2024-02-22 21:29:25 +01:00
2024-03-23 14:53:26 +01:00
std::string expected_result_path;
if (std::filesystem::exists(expectation_path))
{
expected_result_path = expectation_path.string();
}
else if (std::filesystem::exists(failures_path))
{
expected_result_path = failures_path.string();
}
else
{
return test_result{ test_status::expectation_not_found, run.code, run.output };
}
2024-03-14 08:52:45 +01:00
boost::process::opstream pipe_stream;
2024-02-22 21:29:25 +01:00
boost::process::child diff(
2024-03-14 08:52:45 +01:00
boost::process::search_path("diff"), "-Nur", "--color",
2024-03-23 14:53:26 +01:00
expected_result_path, "-",
2024-02-22 21:29:25 +01:00
boost::process::std_in < pipe_stream
);
2024-03-14 08:52:45 +01:00
pipe_stream << run.output;
pipe_stream.flush();
pipe_stream.pipe().close();
2024-02-22 21:29:25 +01:00
diff.wait();
2024-03-14 08:52:45 +01:00
if (diff.exit_code() == 0)
{
2024-03-23 14:53:26 +01:00
return test_result{ test_status::successful, run.code, run.output };
2024-03-14 08:52:45 +01:00
}
return test_result{ test_status::expectation_failed, run.code, run.output };
}
2024-03-23 14:53:26 +01:00
test_result run_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
2024-03-14 08:52:45 +01:00
{
const std::filesystem::path test_filename = test_entry.path().filename();
std::filesystem::path test_binary = in_build_directory(test_filename);
test_binary.replace_extension();
2024-03-23 14:53:26 +01:00
return run_for_output(context,
boost::process::search_path("qemu-riscv32"),
{ test_binary.string() });
2024-02-22 21:29:25 +01:00
}
2024-02-18 11:26:57 +01:00
2024-02-22 21:29:25 +01:00
static test_results run_in_path(const std::filesystem::path test_directory)
2024-02-18 11:26:57 +01:00
{
2024-02-22 21:29:25 +01:00
test_results results;
2024-03-23 14:53:26 +01:00
boost::asio::io_context context;
2024-02-22 21:29:25 +01:00
for (const auto& test_entry : std::filesystem::directory_iterator(test_directory))
2024-02-18 11:26:57 +01:00
{
2024-02-22 21:29:25 +01:00
if (test_entry.path().extension() != ".eln" || !test_entry.is_regular_file())
{
continue;
}
2024-03-14 08:52:45 +01:00
test_result result;
2024-02-25 15:16:19 +01:00
2024-03-14 08:52:45 +01:00
std::cout << "Running " << test_entry << std::endl;
2024-02-25 15:16:19 +01:00
try
{
2024-03-23 14:53:26 +01:00
result = build_test(context, test_entry);
2024-03-14 08:52:45 +01:00
if (result.status == test_status::successful)
{
2024-03-23 14:53:26 +01:00
result = run_test(context, test_entry);
2024-03-14 08:52:45 +01:00
}
result = check_expectation(test_entry, result);
2024-02-25 15:16:19 +01:00
}
catch (const boost::process::process_error& exception)
{
2024-03-14 08:52:45 +01:00
result.status = test_status::build_failed;
result.output = exception.what();
std::cout << result.output << std::endl;
2024-02-25 15:16:19 +01:00
}
2024-03-14 08:52:45 +01:00
print_result(test_entry, result);
results.add_exit_code(result.status);
2024-02-18 11:26:57 +01:00
}
2024-02-22 21:29:25 +01:00
return results;
2024-02-18 11:26:57 +01:00
}
2024-03-14 08:52:45 +01:00
void print_result(const std::filesystem::directory_entry& test_entry, const test_result& result)
{
if (result.status != test_status::successful)
{
std::cout << test_entry << " failed." << std::endl;
}
}
2024-02-22 21:29:25 +01:00
};
2024-02-18 11:26:57 +01:00
int main()
{
2024-03-03 13:11:39 +01:00
std::filesystem::create_directory(elna::in_build_directory());
2024-02-18 11:26:57 +01:00
std::cout << "Run all tests and check the results" << std::endl;
std::filesystem::path test_directory{ "tests" };
2024-02-22 21:29:25 +01:00
const auto results = elna::run_in_path(test_directory);
2024-02-18 11:26:57 +01:00
std::cout << std::endl;
std::cout << results.total() << " tests run, "
<< results.passed() << " passed, "
<< results.failed() << " failed." << std::endl;
return results.exit_code();
}