elna/tests/tester.cpp

#include "elna/tester.hpp"

#include <future>
#include <iostream>

#include <boost/process.hpp>
#include <boost/asio.hpp>

namespace elna
{
    std::uint32_t test_results::total() const noexcept
    {
        return m_total;
    }

    std::uint32_t test_results::passed() const noexcept
    {
        return m_passed;
    }

    std::uint32_t test_results::failed() const noexcept
    {
        return m_total - m_passed;
    }

    int test_results::exit_code() const noexcept
    {
        return m_total == m_passed ? EXIT_SUCCESS : EXIT_FAILURE;
    }

    void test_results::add_exit_code(const test_status status) noexcept
    {
        ++m_total;
        if (status == test_status::successful)
        {
            ++m_passed;
        }
    }

    static std::filesystem::path in_build_directory()
    {
        return "build/riscv";
    }

    static std::string in_build_directory(const std::filesystem::path& path)
    {
        return in_build_directory() / path;
    }

    static test_result build_test(const std::filesystem::directory_entry& test_entry)
    {
        const std::filesystem::path test_filename = test_entry.path().filename();

        std::filesystem::path test_binary = in_build_directory(test_filename);
        std::filesystem::path test_object = test_binary;
        test_binary.replace_extension();
        test_object.replace_extension(".o");

        std::filesystem::remove(test_binary);
        std::filesystem::remove(test_object);

        auto status = boost::process::system("./build/bin/elna",
                "-o", test_object.string(), test_entry.path().string());
        if (status != 0)
        {
            return test_result{ test_status::compile_failed, status };
        }
        status = boost::process::system(
                "/opt/riscv/bin/riscv32-unknown-elf-ld",
                "-o", test_binary.string(),
                "-L/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/",
                "-L/opt/riscv/riscv32-unknown-elf/lib",
                "/opt/riscv/riscv32-unknown-elf/lib/crt0.o",
                "/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtbegin.o",
               test_object.string(),
                "--start-group", "-lgcc", "-lc", "-lgloss", "--end-group",
                "/opt/riscv/lib/gcc/riscv32-unknown-elf/13.2.0/crtend.o"
        );
        return test_result{};
    }

    static test_result check_expectation(const std::filesystem::directory_entry& test_entry, const test_result& run)
    {
        const std::filesystem::path test_filename = test_entry.path().filename();

        std::filesystem::path expectation_path = test_entry.path().parent_path() / "expectations" / test_filename;
        expectation_path.replace_extension(".txt");

        boost::process::opstream pipe_stream;
        boost::process::child diff(
                boost::process::search_path("diff"), "-Nur", "--color",
                expectation_path.string(), "-",
                boost::process::std_in < pipe_stream
        );

        pipe_stream << run.output;
        pipe_stream.flush();
        pipe_stream.pipe().close();
        diff.wait();

        if (diff.exit_code() == 0)
        {
            return run;
        }
        return test_result{ test_status::expectation_failed, run.code, run.output };
    }

    static test_result run_test(const std::filesystem::directory_entry& test_entry)
    {
        const std::filesystem::path test_filename = test_entry.path().filename();

        std::filesystem::path test_binary = in_build_directory(test_filename);
        test_binary.replace_extension();

        boost::asio::io_service io_service;
        std::future<std::string> buffer;
        boost::process::child spike(
            "/opt/riscv/bin/spike", "--isa=RV32IMAC",
            "/opt/riscv/riscv32-unknown-elf/bin/pk",
            test_binary.string(),
            boost::process::std_out > buffer,
            boost::process::std_err > buffer,
            io_service
        );
        io_service.run();

        return test_result{ test_status::successful, spike.exit_code(), buffer.get() };
    }

    static test_results run_in_path(const std::filesystem::path test_directory)
    {
        test_results results;

        for (const auto& test_entry : std::filesystem::directory_iterator(test_directory))
        {
            if (test_entry.path().extension() != ".eln" || !test_entry.is_regular_file())
            {
                continue;
            }
            test_result result;

            std::cout << "Running " << test_entry << std::endl;
            try
            {
                result = build_test(test_entry);
                if (result.status == test_status::successful)
                {
                    result = run_test(test_entry);
                }
                result = check_expectation(test_entry, result);
            }
            catch (const boost::process::process_error& exception)
            {
                result.status = test_status::build_failed;
                result.output = exception.what();
                std::cout << result.output << std::endl;
            }
            print_result(test_entry, result);
            results.add_exit_code(result.status);
        }
        return results;
    }

    void print_result(const std::filesystem::directory_entry& test_entry, const test_result& result)
    {
        if (result.status != test_status::successful)
        {
            std::cout << test_entry << " failed." << std::endl;
        }
    }
};

int main()
{
    std::filesystem::create_directory(elna::in_build_directory());

    std::cout << "Run all tests and check the results" << std::endl;
    std::filesystem::path test_directory{ "tests" };
    const auto results = elna::run_in_path(test_directory);

    std::cout << std::endl;
    std::cout << results.total() << " tests run, "
        << results.passed() << " passed, "
        << results.failed() << " failed." << std::endl;

    return results.exit_code();
}