belka/elna
1
0
Fork 0
Code Issues 5 Pull Requests Releases Activity

Support failure tests

Browse Source
This commit is contained in:
Eugen Wissner
2024-03-23 14:53:26 +01:00
parent a78e08521e
commit 30f80bcb88
14 changed files with 241 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/expectations/procedure_definition.txt Normal file
View File

@ -0,0 +1 @@
2

1
tests/failures/single_word_error.txt Normal file
View File

@ -0,0 +1 @@
1:1: Unexpected token «identifier»

6
tests/procedure_definition.eln Normal file
View File

@ -0,0 +1,6 @@
proc f;
writei(5);
begin
writei(2)
end.

1
tests/single_word_error.eln Normal file
View File

@ -0,0 +1 @@
asdf

117
tests/tester.cpp
View File

@ -1,11 +1,7 @@
#include "elna/tester.hpp"
#include <future>
#include <iostream>
#include <boost/process.hpp>
#include <boost/asio.hpp>
namespace elna
{
std::uint32_t test_results::total() const noexcept
@ -47,7 +43,36 @@ namespace elna
return in_build_directory() / path;
}
static test_result build_test(const std::filesystem::directory_entry& test_entry)
test_result run_for_output(boost::asio::io_context& context, const std::filesystem::path& binary,
std::initializer_list<boost::string_view> arguments)
{
boost::asio::readable_pipe read_pipe{ context };
test_result result{};
std::string output;
boost::asio::dynamic_string_buffer buffer = boost::asio::dynamic_buffer(output);
boost::system::error_code ec;
boost::process::v2::process elna_child(context,
binary, arguments,
boost::process::v2::process_stdio{ nullptr, read_pipe, read_pipe });
do
{
std::size_t transferred = read_pipe.read_some(buffer.prepare(512), ec);
if (transferred == 0)
{
break;
}
buffer.commit(transferred);
result.output.append(boost::asio::buffer_cast<const char *>(buffer.data()), buffer.size());
buffer.consume(transferred);
}
while (ec == boost::system::errc::success);
result.code = elna_child.wait();
return result;
}
test_result build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
{
const std::filesystem::path test_filename = test_entry.path().filename();
@ -59,37 +84,57 @@ namespace elna
std::filesystem::remove(test_binary);
std::filesystem::remove(test_object);
auto status = boost::process::system("./build/bin/elna",
"-o", test_object.string(), test_entry.path().string());
if (status != 0)
test_result result = run_for_output(context, "./build/bin/elna",
{ "-o", test_object.string(), test_entry.path().string() });
if (result.code != 0)
{
return test_result{ test_status::compile_failed, status };
result.status = test_status::compile_failed;
return result;
}
status = boost::process::system(
"/opt/riscv/bin/riscv32-unknown-elf-ld",
"-o", test_binary.string(),
"-L/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/",
"-L/opt/riscv/riscv32-unknown-elf/lib",
"/opt/riscv/riscv32-unknown-elf/lib/crt0.o",
"/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtbegin.o",
test_object.string(),
"--start-group", "-lgcc", "-lc", "-lgloss", "--end-group",
"/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtend.o"
);
boost::process::v2::execute(boost::process::v2::process(
context, "/opt/riscv/bin/riscv32-unknown-elf-ld",
{
"-o", test_binary.string(),
"-L/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/",
"-L/opt/riscv/riscv32-unknown-elf/lib",
"/opt/riscv/riscv32-unknown-elf/lib/crt0.o",
"/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtbegin.o",
test_object.string(),
"--start-group", "-lgcc", "-lc", "-lgloss", "--end-group",
"/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtend.o"
}
));
return test_result{};
}
static test_result check_expectation(const std::filesystem::directory_entry& test_entry, const test_result& run)
{
const std::filesystem::path test_filename = test_entry.path().filename();
std::filesystem::path test_filename = test_entry.path().filename();
test_filename.replace_extension(".txt");
std::filesystem::path expectation_path = test_entry.path().parent_path() / "expectations" / test_filename;
expectation_path.replace_extension(".txt");
const std::filesystem::path expectation_path =
test_entry.path().parent_path() / "expectations" / test_filename;
const std::filesystem::path failures_path =
test_entry.path().parent_path() / "failures" / test_filename;
std::string expected_result_path;
if (std::filesystem::exists(expectation_path))
{
expected_result_path = expectation_path.string();
}
else if (std::filesystem::exists(failures_path))
{
expected_result_path = failures_path.string();
}
else
{
return test_result{ test_status::expectation_not_found, run.code, run.output };
}
boost::process::opstream pipe_stream;
boost::process::child diff(
boost::process::search_path("diff"), "-Nur", "--color",
expectation_path.string(), "-",
expected_result_path, "-",
boost::process::std_in < pipe_stream
);
@ -100,35 +145,27 @@ namespace elna
if (diff.exit_code() == 0)
{
return run;
return test_result{ test_status::successful, run.code, run.output };
}
return test_result{ test_status::expectation_failed, run.code, run.output };
}
static test_result run_test(const std::filesystem::directory_entry& test_entry)
test_result run_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
{
const std::filesystem::path test_filename = test_entry.path().filename();
std::filesystem::path test_binary = in_build_directory(test_filename);
test_binary.replace_extension();
boost::asio::io_service io_service;
std::future<std::string> buffer;
boost::process::child qemu(
boost::process::search_path("qemu-riscv32"),
test_binary.string(),
boost::process::std_out > buffer,
boost::process::std_err > buffer,
io_service
);
io_service.run();
return test_result{ test_status::successful, qemu.exit_code(), buffer.get() };
return run_for_output(context,
boost::process::search_path("qemu-riscv32"),
{ test_binary.string() });
}
static test_results run_in_path(const std::filesystem::path test_directory)
{
test_results results;
boost::asio::io_context context;
for (const auto& test_entry : std::filesystem::directory_iterator(test_directory))
{
@ -141,10 +178,10 @@ namespace elna
std::cout << "Running " << test_entry << std::endl;
try
{
result = build_test(test_entry);
result = build_test(context, test_entry);
if (result.status == test_status::successful)
{
result = run_test(test_entry);
result = run_test(context, test_entry);
}
result = check_expectation(test_entry, result);
}