elna/tests/tester.cpp

#include "elna/tester.hpp"

#include <iostream>
#include <fstream>
#include <algorithm>
#include <boost/process/env.hpp>

namespace elna
{
    std::uint32_t test_results::total() const noexcept
    {
        return m_total;
    }

    std::uint32_t test_results::passed() const noexcept
    {
        return m_passed;
    }

    std::uint32_t test_results::failed() const noexcept
    {
        return m_total - m_passed;
    }

    int test_results::exit_code() const noexcept
    {
        return m_total == m_passed ? EXIT_SUCCESS : EXIT_FAILURE;
    }

    void test_results::add_exit_code(const test_status status) noexcept
    {
        ++m_total;
        if (status == test_status::successful)
        {
            ++m_passed;
        }
    }

    static std::filesystem::path in_build_directory()
    {
        return "build/riscv";
    }

    static std::string in_build_directory(const std::filesystem::path& path)
    {
        return in_build_directory() / path;
    }

    static std::filesystem::path in_actual_directory()
    {
        return "build/tests";
    }

    boost::process::v2::process_stdio get_output_streams(const std::uint8_t stream_number,
            boost::asio::readable_pipe& read_pipe)
    {
        if (stream_number == 1)
        {
            return boost::process::v2::process_stdio{ nullptr, read_pipe };
        }
        if (stream_number == 2)
        {
            return boost::process::v2::process_stdio{ nullptr, {}, read_pipe };
        }
        return boost::process::v2::process_stdio{ nullptr, read_pipe, read_pipe };
    }

    int run_for_output(boost::asio::io_context& context, const std::filesystem::path& log_path,
            const std::filesystem::path& binary, std::initializer_list<boost::string_view> arguments)
    {
        boost::asio::readable_pipe read_pipe{ context };
        std::string output;
        boost::asio::dynamic_string_buffer buffer = boost::asio::dynamic_buffer(output);
        boost::system::error_code ec;

        boost::process::v2::process_stdio process_stdio;
        boost::process::v2::process elna_child(context, binary, arguments,
                boost::process::v2::process_stdio{ nullptr, read_pipe, 1 });
        std::ofstream log_file{ log_path };
        do
        {
            std::size_t transferred = read_pipe.read_some(buffer.prepare(512), ec);

            if (transferred == 0)
            {
                break;
            }
            buffer.commit(transferred);
            log_file.write(boost::asio::buffer_cast<const char *>(buffer.data()), buffer.size());
            buffer.consume(transferred);
        }
        while (ec == boost::system::errc::success);

        return elna_child.wait();
    }

    std::string find_object(const std::vector<std::filesystem::path>& environment, const std::string& object)
    {
        auto variable = std::find(environment.cbegin(), environment.cend(), object);

        for (const auto& variable : environment)
        {
            auto full_path = variable / object;

            if (std::filesystem::exists(full_path))
            {
                return full_path.native();
            }
        }
        return object;
    }

    test_status build_test(boost::asio::io_context& context, const std::filesystem::directory_entry& test_entry)
    {
        const std::filesystem::path test_filename = test_entry.path().filename();

        std::filesystem::path test_binary = in_build_directory(test_filename);
        std::filesystem::path test_object = test_binary;
        std::filesystem::path test_log = test_binary;
        test_binary.replace_extension();
        test_object.replace_extension(".o");
        test_log.replace_extension(".log");

        std::filesystem::remove(test_binary);
        std::filesystem::remove(test_object);

        auto result_code = run_for_output(context, test_log, "./build/bin/elna",
                { "-o", test_object.string(), test_entry.path().string() });
        if (result_code != 0)
        {
            return test_status::compile_failed;
        }
        test_status result = result_code == 0 ? test_status::successful : test_status::compile_failed;
        std::vector<std::filesystem::path> environment;
        std::vector<std::string> linker_arguments = { "-o", test_binary.string() };

        for (const auto segment : boost::this_process::environment()["LIBRARY_PATH"].to_vector())
        {
            linker_arguments.push_back("-L" + segment);
            environment.push_back(std::filesystem::path(segment));
        }
        linker_arguments.push_back(find_object(environment, "crt0.o"));
        linker_arguments.push_back(find_object(environment, "crtbegin.o"));
        linker_arguments.push_back(test_object.string());
        linker_arguments.insert(linker_arguments.cend(),
                { "--start-group", "-lgcc", "-lc", "-lgloss", "--end-group" });
        linker_arguments.push_back(find_object(environment, "crtend.o"));

        boost::process::v2::execute(boost::process::v2::process(context, boost::process::search_path("ld"),
                    linker_arguments
        ));
        return result;
    }

    static test_status check_expectation(const std::filesystem::path& expectation_path,
            const std::filesystem::path& actual_directory)
    {
        std::filesystem::path log_filename = expectation_path.filename();
        log_filename.replace_extension(".log");

        std::filesystem::path test_log = actual_directory / log_filename;

        test_log.replace_extension(".log");
        boost::process::child diff(
                boost::process::search_path("diff"), "-Nur", "--color",
                expectation_path, test_log
        );
        diff.wait();

        if (diff.exit_code() == 0)
        {
            return test_status::successful;
        }
        return test_status::expectation_failed;
    }

    void run_test(boost::asio::io_context& context, const std::filesystem::path& test_binary)
    {
        std::filesystem::path test_log = test_binary;
        test_log.replace_extension(".log");

        run_for_output(context, test_log, boost::process::search_path("qemu-riscv32"),
                { test_binary.string() });
    }

    static test_results run_in_path(const std::filesystem::path test_directory)
    {
        test_results results;
        boost::asio::io_context context;

        for (const auto& test_entry : std::filesystem::directory_iterator(test_directory))
        {
            if (test_entry.path().extension() != ".eln" || !test_entry.is_regular_file())
            {
                continue;
            }
            test_status result;

            std::cout << "Running " << test_entry << std::endl;
            try
            {
                build_test(context, test_entry);
            }
            catch (const boost::process::process_error& exception)
            {
                test_status::build_failed;
                std::cout << exception.what() << std::endl;
            }
        }
        for (const auto& executable_test : std::filesystem::directory_iterator(in_build_directory()))
        {
            if (executable_test.path().has_extension())
            {
                continue;
            }
            run_test(context, executable_test.path());
        }
        for (const auto& expectation_entry : std::filesystem::directory_iterator(test_directory / "failures"))
        {
            auto test_entry = test_directory / expectation_entry.path().filename();
            test_entry.replace_extension(".eln");
            auto result = check_expectation(expectation_entry.path(), in_build_directory());

            print_result(test_entry, result);
            results.add_exit_code(result);
        }
        for (const auto& expectation_entry : std::filesystem::directory_iterator(test_directory / "expectations"))
        {
            auto test_entry = test_directory / expectation_entry.path().filename();
            test_entry.replace_extension(".eln");
            auto result = check_expectation(expectation_entry.path(), in_actual_directory());

            print_result(test_entry, result);
            results.add_exit_code(result);
        }
        return results;
    }

    void print_result(const std::filesystem::path& test_entry, const test_status& result)
    {
        if (result != test_status::successful)
        {
            std::cout << test_entry << " failed." << std::endl;
        }
    }
};

int main()
{
    std::filesystem::create_directory(elna::in_build_directory());
    std::filesystem::create_directory(elna::in_actual_directory());

    std::cout << "Run all tests and check the results" << std::endl;
    std::filesystem::path test_directory{ "tests" };
    const auto results = elna::run_in_path(test_directory);

    std::cout << std::endl;
    std::cout << results.total() << " tests run, "
        << results.passed() << " passed, "
        << results.failed() << " failed." << std::endl;

    return results.exit_code();
}