Add scripts to build a toolchain for running VM tests

This commit is contained in:
Eugen Wissner 2024-05-14 22:09:05 +02:00
parent 51fa3efc32
commit 68264016ce
Signed by: belka
GPG Key ID: A27FDC1E8EE902C0
10 changed files with 688 additions and 15 deletions

3
TODO
View File

@ -12,3 +12,6 @@
- Syscalls.
- Error message with an empty file wrongly says that a ")" is expected.
- Support any expressions for constants.
# Toolchain
- Try to guess the build platform (x86_64-slackware-linux is hard coded).

View File

@ -16,9 +16,7 @@ namespace elna
{
successful,
compile_failed,
build_failed,
expectation_failed,
expectation_not_found
};
class test_results final

12
package.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "elna",
"version": "1.0.0",
"description": "",
"main": "tools/index.js",
"type": "module",
"scripts": {
"start": "node tools/index.js"
},
"author": "Eugen Wissner <belka@caraus.de>",
"license": "MPL-2.0"
}

View File

@ -1,5 +1,6 @@
#include "elna/tester.hpp"
#include <filesystem>
#include <iostream>
#include <fstream>
#include <algorithm>
@ -51,6 +52,16 @@ namespace elna
return "build/tests";
}
static std::filesystem::path in_root_directory()
{
return "build/root";
}
static std::filesystem::path in_root_directory(const std::filesystem::path& path)
{
return in_root_directory() / path;
}
boost::process::v2::process_stdio get_output_streams(const std::uint8_t stream_number,
boost::asio::readable_pipe& read_pipe)
{
@ -114,9 +125,9 @@ namespace elna
{
const std::filesystem::path test_filename = test_entry.path().filename();
std::filesystem::path test_binary = in_build_directory(test_filename);
std::filesystem::path test_object = test_binary;
std::filesystem::path test_log = test_binary;
std::filesystem::path test_binary = in_root_directory("tests" / test_filename);
std::filesystem::path test_object = in_build_directory(test_filename);
std::filesystem::path test_log = in_build_directory(test_filename);
test_binary.replace_extension();
test_object.replace_extension(".o");
test_log.replace_extension(".log");
@ -183,6 +194,74 @@ namespace elna
{ test_binary.string() });
}
static void cpio_archive(boost::asio::io_context& context)
{
boost::asio::readable_pipe read_pipe{ context };
boost::asio::writable_pipe write_pipe{ context };
std::string output;
boost::asio::dynamic_string_buffer buffer = boost::asio::dynamic_buffer(output);
boost::system::error_code ec;
std::ofstream archive_output{ "build/root.cpio", std::ios::binary };
boost::process::v2::process_stdio process_stdio;
auto current_path = std::filesystem::current_path();
std::filesystem::current_path(in_root_directory());
boost::process::v2::process cpio_child(context, boost::process::search_path("cpio"),
{ "-o", "--format=newc" },
boost::process::v2::process_stdio{ write_pipe, read_pipe, nullptr });
for (const auto& entry : std::filesystem::recursive_directory_iterator("."))
{
auto entry_path = entry.path().string() + "\n";
auto entry_iterator = std::cbegin(entry_path);
std::size_t written{ 0 };
while (entry_iterator != std::cend(entry_path))
{
std::size_t written = write_pipe.write_some(boost::asio::buffer(entry_iterator.base(),
std::distance(entry_iterator, std::cend(entry_path))));
std::advance(entry_iterator, written);
}
}
write_pipe.close();
do
{
std::size_t transferred = read_pipe.read_some(buffer.prepare(512), ec);
if (transferred == 0)
{
break;
}
buffer.commit(transferred);
archive_output.write(boost::asio::buffer_cast<const char *>(buffer.data()), buffer.size());
buffer.consume(transferred);
}
while (ec == boost::system::errc::success);
std::filesystem::current_path(current_path);
}
static void run_vm(boost::asio::io_context& context)
{
std::vector<std::string> arguments = {
"-nographic",
"-M", "virt",
"-bios", "default",
"-kernel", "build/tools/linux-5.15.158/arch/riscv/boot/Image",
"-append", "quiet",
"-initrd", "build/root.cpio"
};
auto process = boost::process::v2::process(context,
boost::process::search_path("qemu-system-riscv32"), arguments);
boost::process::v2::execute(std::move(process));
}
static void create_init()
{
std::filesystem::copy("build/tools/init", in_root_directory());
}
static test_results run_in_path(const std::filesystem::path test_directory)
{
test_results results;
@ -196,14 +275,13 @@ namespace elna
}
test_status result;
std::cout << "Running " << test_entry << std::endl;
std::cout << "Compiling " << test_entry << std::endl;
try
{
build_test(context, test_entry);
}
catch (const boost::process::process_error& exception)
{
test_status::build_failed;
std::cout << exception.what() << std::endl;
}
}
@ -224,15 +302,12 @@ namespace elna
print_result(test_entry, result);
results.add_exit_code(result);
}
for (const auto& expectation_entry : std::filesystem::directory_iterator(test_directory / "expectations"))
{
auto test_entry = test_directory / expectation_entry.path().filename();
test_entry.replace_extension(".eln");
auto result = check_expectation(expectation_entry.path(), in_actual_directory());
std::filesystem::copy(test_directory / "expectations", in_root_directory("expectations"),
std::filesystem::copy_options::recursive);
create_init();
cpio_archive(context);
run_vm(context);
print_result(test_entry, result);
results.add_exit_code(result);
}
return results;
}
@ -250,6 +325,14 @@ int main()
std::filesystem::create_directory(elna::in_build_directory());
std::filesystem::create_directory(elna::in_actual_directory());
std::filesystem::remove_all(elna::in_root_directory());
std::filesystem::create_directory(elna::in_root_directory());
std::filesystem::create_directory(elna::in_root_directory("expectations"));
std::filesystem::create_directory(elna::in_root_directory("tests"));
auto current_environment = boost::this_process::environment();
current_environment["PATH"] += "./build/tools/sysroot/bin";
std::cout << "Run all tests and check the results" << std::endl;
std::filesystem::path test_directory{ "tests" };
const auto results = elna::run_in_path(test_directory);

2
tools/files/fstab Normal file
View File

@ -0,0 +1,2 @@
/dev/vda / ext4 defaults 1 1
proc /proc proc defaults 0 0

3
tools/files/inittab Normal file
View File

@ -0,0 +1,3 @@
::sysinit:/etc/init.d/rcS
::shutdown:/etc/init.d/rcK
::askfirst:-/bin/sh

7
tools/files/rcK Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
PATH=/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/bin:/bin
killall5 -15
sleep 2
umount -v -a -r -t no,proc,sysfs,devtmpfs

7
tools/files/rcS Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
PATH=/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/bin:/bin
mount -v proc /proc -n -t proc
mount -w -v -n -o remount /
mount -a

373
tools/index.js Normal file
View File

@ -0,0 +1,373 @@
import fs from 'fs/promises'
import path from 'node:path'
import childProcess from 'node:child_process'
import process from 'process'
import os from 'os'
// Define constants.
const tmp = path.resolve('./build/tools')
const target = 'riscv32-unknown-linux-gnu'
const build = 'x86_64-slackware-linux'
const baseDirectory = path.resolve('./tools')
const busyboxVersion = '1.36.1'
const kernelVersion = '5.15.158'
const gccVersion = '13.2.0'
const binutilsVersion = '2.42'
const glibcVersion = '2.39'
function createImage (rootfs) {
const rootExt4 = path.join(tmp, 'rootfs.ext4')
childProcess.execFileSync('dd', ['if=/dev/zero', `of=${rootExt4}`, 'bs=1M', 'count=1024'], { stdio: 'inherit' })
childProcess.execFileSync('/sbin/mkfs.ext4', ['-d', rootfs, rootExt4], { stdio: 'inherit' })
}
async function downloadAndUnarchive (url) {
const response = await fetch(url)
const bodyReader = response.body.getReader()
const basename = path.basename(url.pathname)
let archiveType = ''
let rootDirectory = ''
switch (path.extname(basename)) {
case '.bz2':
archiveType = '-j'
rootDirectory = path.basename(basename, '.tar.bz2')
break
case '.xz':
archiveType = '-J'
rootDirectory = path.basename(basename, '.tar.xz')
break
default:
break
}
return new Promise(async function (resolve, reject) {
const untar = childProcess.spawn('tar', ['-C', tmp, archiveType, '-xv'], { stdio: ['pipe', 'inherit', 'inherit'] })
let done = false
untar.on('exit', function () {
resolve(path.join(tmp, rootDirectory))
})
do {
const chunk = await bodyReader.read()
done = chunk.done
if (chunk.value !== undefined) {
untar.stdin.write(chunk.value)
}
} while (!done)
untar.stdin.end()
})
}
async function buildBusyBox (sysroot, rootfs) {
const cwd = await downloadAndUnarchive(new URL(`https://busybox.net/downloads/busybox-${busyboxVersion}.tar.bz2`))
const env = {
...process.env,
CROSS_COMPILE: path.join(sysroot, 'bin', `${target}-`)
}
const configuration = [
`CONFIG_PREFIX=${rootfs}`,
`CONFIG_SYSROOT=${rootfs}`
]
childProcess.execFileSync('make', ['defconfig'], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', [...configuration, 'install'], { stdio: 'inherit', env, cwd })
}
async function buildKernel (sysroot, rootfs) {
const cwd = await downloadAndUnarchive(
new URL(`https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-${kernelVersion}.tar.xz`)
)
const env = {
...process.env,
CROSS_COMPILE: `${target}-`,
ARCH: 'riscv',
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
childProcess.execFileSync('make', ['rv32_defconfig'], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', ['headers'], { stdio: 'inherit', env, cwd })
const userDirectory = path.join(rootfs, 'usr')
await fs.cp(path.join(cwd, 'usr/include'), path.join(userDirectory, 'include'), { recursive: true })
return path.join(cwd, 'arch/riscv/boot/Image')
}
async function buildInit (sysroot) {
const env = {
...process.env,
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
const compilerArguments = [
'-ffreestanding', '-static',
'-o', path.join(tmp, 'init'),
path.join(baseDirectory, 'init.cpp')
]
childProcess.execFileSync('riscv32-unknown-linux-gnu-g++', compilerArguments, { stdio: 'inherit', env })
}
async function buildGlibc (sysroot, rootfs) {
const sourceDirectory = await downloadAndUnarchive(
new URL(`https://ftp.gnu.org/gnu/glibc/glibc-${glibcVersion}.tar.xz`)
)
const configureOptions = [
'--prefix=/usr',
'--libdir=/usr/lib',
`--host=${target}`,
`--build=${build}`,
`--enable-kernel=${kernelVersion}`,
`--with-headers=${path.join(rootfs, 'usr/include')}`,
'--disable-nscd'
]
const env = {
...process.env,
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
const cwd = path.join(path.dirname(sourceDirectory), 'build-glibc');
await fs.mkdir(cwd)
childProcess.execFileSync(path.join(path.relative(cwd, sourceDirectory), './configure'), configureOptions, {
stdio: 'inherit',
env,
cwd
})
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', [`DESTDIR=${rootfs}`, 'install'], { stdio: 'inherit', env, cwd })
}
async function buildCrossBinutils (sysroot) {
const sourceDirectory = await downloadAndUnarchive(
new URL(`https://ftp.gnu.org/gnu/binutils/binutils-${binutilsVersion}.tar.xz`)
)
const cwd = path.join(path.dirname(sourceDirectory), 'build-binutils');
await fs.mkdir(cwd)
const configureOptions = [
`--prefix=${sysroot}`,
`--target=${target}`,
'--disable-nls',
'--enable-gprofng=no',
'--disable-werror',
'--enable-default-hash-style=gnu'
]
childProcess.execFileSync(path.join(path.relative(cwd, sourceDirectory), 'configure'), configureOptions, {
stdio: 'inherit',
cwd
})
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', cwd })
childProcess.execFileSync('make', ['install'], { stdio: 'inherit', cwd })
return sourceDirectory
}
async function buildGCC1 (sysroot, rootfs) {
const sourceDirectory = await downloadAndUnarchive(
new URL(`https://download.dlackware.com/slackware/slackware64-current/source/d/gcc/gcc-${gccVersion}.tar.xz`)
)
const cwd = path.join(path.dirname(sourceDirectory), 'build-gcc');
await fs.mkdir(cwd)
childProcess.execFileSync(path.join(sourceDirectory, 'contrib/download_prerequisites'), {
stdio: 'inherit',
cwd: sourceDirectory
})
const configureOptions = [
`--prefix=${sysroot}`,
`--with-sysroot=${rootfs}`,
'--enable-languages=c,c++',
'--disable-shared',
'--with-arch=rv32imafdc',
'--with-abi=ilp32d',
'--with-tune=rocket',
'--with-isa-spec=20191213',
'--disable-bootstrap',
'--disable-multilib',
'--disable-libmudflap',
'--disable-libssp',
'--disable-libquadmath',
'--disable-libsanitizer',
'--disable-threads',
'--disable-libatomic',
'--disable-libgomp',
'--disable-libvtv',
'--disable-libstdcxx',
'--disable-nls',
'--with-newlib',
'--without-headers',
`--target=${target}`,
`--build=${build}`,
`--host=${build}`
]
const flags = '-O2 -fPIC'
const env = {
...process.env,
CFLAGS: flags,
CXXFLAGS: flags,
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
childProcess.execFileSync(path.join(path.relative(cwd, sourceDirectory), 'configure'), configureOptions, {
stdio: 'inherit',
env,
cwd
})
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', ['install'], { stdio: 'inherit', env, cwd })
return sourceDirectory
}
async function buildBinutils (sourceDirectory, sysroot, rootfs) {
const cwd = path.join(path.dirname(sourceDirectory), 'build-binutils');
await fs.rm(cwd, { recursive: true, force: true })
await fs.mkdir(cwd)
const configureOptions = [
'--prefix=/usr',
`--build=${build}`,
`--host=${target}`,
`--with-build-sysroot=${rootfs}`,
'--disable-nls',
'--enable-shared',
'--enable-gprofng=no',
'--disable-werror',
'--enable-default-hash-style=gnu'
]
const env = {
...process.env,
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
childProcess.execFileSync(path.join(path.relative(cwd, sourceDirectory), 'configure'), configureOptions, {
stdio: 'inherit',
env,
cwd
})
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', [`DESTDIR=${rootfs}`, 'install'], { stdio: 'inherit', env, cwd })
return sourceDirectory
}
async function buildGCC2 (sourceDirectory, sysroot, rootfs) {
const cwd = path.join(path.dirname(sourceDirectory), 'build-gcc');
await fs.rm(cwd, { recursive: true, force: true })
await fs.mkdir(cwd)
const configureOptions = [
`--prefix=${sysroot}`,
`--with-sysroot=${rootfs}`,
'--enable-languages=c,c++,lto',
'--enable-lto',
'--enable-shared',
'--with-arch=rv32imafdc',
'--with-abi=ilp32d',
'--with-tune=rocket',
'--with-isa-spec=20191213',
'--disable-bootstrap',
'--disable-multilib',
'--enable-checking=release',
'--disable-libssp',
'--enable-threads=posix',
'--with-default-libstdcxx-abi=new',
'--disable-nls',
`--target=${target}`,
`--build=${build}`,
`--host=${build}`
]
const flags = '-O2 -fPIC'
const env = {
...process.env,
CFLAGS: flags,
CXXFLAGS: flags,
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
childProcess.execFileSync(path.join(path.relative(cwd, sourceDirectory), 'configure'), configureOptions, {
stdio: 'inherit',
env,
cwd
})
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', ['install'], { stdio: 'inherit', env, cwd })
}
async function buildGCC3 (sourceDirectory, sysroot, rootfs) {
const cwd = path.join(path.dirname(sourceDirectory), 'build-gcc');
await fs.rm(cwd, { recursive: true, force: true })
await fs.mkdir(cwd)
const configureOptions = [
`--prefix=/usr`,
`--libdir=/usr/lib`,
'--enable-languages=c,c++,lto',
'--enable-lto',
'--enable-shared',
'--with-arch=rv32imafdc',
'--with-abi=ilp32d',
'--with-tune=rocket',
'--with-isa-spec=20191213',
'--disable-bootstrap',
'--disable-multilib',
'--enable-checking=release',
'--disable-libssp',
'--enable-threads=posix',
'--with-default-libstdcxx-abi=new',
'--disable-nls',
`--target=${target}`,
`--build=${build}`,
`--host=${target}`
]
const flags = '-O2 -fPIC'
const env = {
...process.env,
CFLAGS: flags,
CXXFLAGS: flags,
PATH: `${path.join(sysroot, 'bin')}:${process.env['PATH']}`
}
childProcess.execFileSync(path.join(path.relative(cwd, sourceDirectory), 'configure'), configureOptions, {
stdio: 'inherit',
env,
cwd
})
childProcess.execFileSync('make', ['-j', os.availableParallelism()], { stdio: 'inherit', env, cwd })
childProcess.execFileSync('make', [`DESTDIR=${rootfs}`, 'install'], { stdio: 'inherit', env, cwd })
}
async function createRoot (rootfs) {
for (const directory of ['proc', 'sys', 'dev', 'etc/init.d', 'usr/local', 'usr/local/bin']) {
await fs.mkdir(path.join(rootfs, directory))
}
await fs.cp(path.join(baseDirectory, 'files/fstab'), path.join(rootfs, 'etc/fstab'))
await fs.cp(path.join(baseDirectory, 'files/rcS'), path.join(rootfs, 'etc/init.d/rcS'))
await fs.cp(path.join(baseDirectory, 'files/rcK'), path.join(rootfs, 'etc/init.d/rcK'))
await fs.cp(path.join(baseDirectory, 'files/inittab'), path.join(rootfs, 'etc/inittab'))
}
const sysroot = path.join(tmp, 'sysroot')
const rootfs = path.join(tmp, 'rootfs')
for (const targetDirectory of [tmp, sysroot, rootfs]) {
await fs.rm(targetDirectory, { recursive: true, force: true })
await fs.mkdir(targetDirectory)
}
const binutilsSource = await buildCrossBinutils(sysroot)
const gccSource = await buildGCC1(sysroot, rootfs)
const kernelImage = await buildKernel(sysroot, rootfs)
await buildGlibc(sysroot, rootfs)
await buildBinutils(binutilsSource, sysroot, rootfs)
await buildGCC2(gccSource, sysroot, rootfs)
buildInit (sysroot)
await buildBusyBox(sysroot, rootfs)
await createRoot(rootfs)
await buildGCC3(gccSource, sysroot, rootfs)
createImage(rootfs)
console.log(kernelImage)

185
tools/init.cpp Normal file
View File

@ -0,0 +1,185 @@
#include <algorithm>
#include <memory>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <sys/reboot.h>
template<std::size_t size>
std::size_t read_command(int descriptor, char *command_buffer)
{
std::ptrdiff_t bytes_read{ 0 };
std::size_t read_so_far{ 0 };
while ((bytes_read = read(descriptor, command_buffer + read_so_far, size - read_so_far - 1)) > 0)
{
read_so_far += bytes_read;
if (read_so_far >= size - 1)
{
break;
}
}
command_buffer[read_so_far] = 0;
return read_so_far;
}
enum class status
{
success,
failure,
warning,
fatal
};
template<std::size_t size>
void make_path(char *destination, const char *directory, const char *filename, const char *extension)
{
memcpy(destination, directory, strlen(directory));
std::size_t remaining_space = size - strlen(directory);
if (extension != nullptr)
{
remaining_space -= strlen(extension) + 1;
}
strncpy(destination + strlen(directory), filename, remaining_space);
if (extension != nullptr)
{
strncpy(destination + strlen(destination), extension, strlen(extension));
}
}
status run_test(const char *file_entry_name)
{
printf("Running %s. ", file_entry_name);
char filename[256];
char command_buffer[256];
char file_buffer[256];
int pipe_ends[2];
if (pipe(pipe_ends) == -1)
{
perror("pipe");
return status::fatal;
}
make_path<sizeof(filename)>(filename, "./tests/", file_entry_name, nullptr);
auto child_pid = fork();
if (child_pid == -1)
{
return status::fatal;
}
else if (child_pid == 0)
{
close(STDIN_FILENO);
close(STDERR_FILENO);
close(pipe_ends[0]); // Close the read end.
if (dup2(pipe_ends[1], STDOUT_FILENO) == -1)
{
perror("dup2");
}
else
{
execl(filename, filename);
perror("execl");
}
close(STDOUT_FILENO);
close(pipe_ends[1]);
std::exit(1);
}
else
{
close(pipe_ends[1]); // Close the write end.
auto read_from_command = read_command<sizeof(command_buffer)>(pipe_ends[0], command_buffer);
close(pipe_ends[0]);
int wait_status{ 0 };
wait(&wait_status);
make_path<sizeof(filename)>(filename, "./expectations/", file_entry_name, ".txt");
std::unique_ptr<FILE, int(*)(FILE *)> expectation_descriptor(fopen(filename, "r"), &fclose);
if (expectation_descriptor == nullptr)
{
return status::warning;
}
auto read_from_file = fread(file_buffer, 1, sizeof(file_buffer) - 1, expectation_descriptor.get());
file_buffer[std::max<std::ptrdiff_t>(0, read_from_file)] = 0;
if (strcmp(command_buffer, file_buffer) != 0)
{
printf("Failed. Got:\n%s", command_buffer);
return status::failure;
}
else
{
fwrite("\n", 1, 1, stdout);
return status::success;
}
}
}
struct summary
{
std::size_t total{ 0 };
std::size_t failure{ 0 };
std::size_t success{ 0 };
};
void walk()
{
auto directory_stream = opendir("./tests");
dirent *file_entry;
summary test_summary;
while ((file_entry = readdir(directory_stream)) != nullptr)
{
if (file_entry->d_name[0] == '.')
{
continue;
}
++test_summary.total;
switch (run_test(file_entry->d_name))
{
case status::failure:
++test_summary.failure;
break;
case status::success:
++test_summary.success;
break;
case status::warning:
break;
case status::fatal:
goto end_walk;
}
}
printf("Successful: %lu, Failed: %lu, Total: %lu.\n",
test_summary.success, test_summary.failure, test_summary.total);
end_walk:
closedir(directory_stream);
}
int main()
{
auto dev_console = open("/dev/console", O_WRONLY);
if (dev_console != -1)
{
dup2(dev_console, STDOUT_FILENO);
walk();
close(dev_console);
}
sync();
reboot(RB_POWER_OFF);
return 1;
}