XFESystem

"""Build pinned musl's mathematical library, without its OS or libc runtime."""
import concurrent.futures
import hashlib
import json
from pathlib import Path
import re
import subprocess
import tarfile
import time
import urllib.request

ROOT = Path(__file__).resolve().parents[3]
ARCHIVE_SHA = 'a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4'
URL = 'https://musl.libc.org/releases/musl-1.2.5.tar.gz'


def digest(path):
    return hashlib.sha256(path.read_bytes()).hexdigest()


def main():
    archive = ROOT / '.tools/musl-1.2.5.tar.gz'
    if not archive.is_file():
        with urllib.request.urlopen(URL, timeout=60) as source:
            data = source.read(16 * 1024 * 1024 + 1)
        assert len(data) <= 16 * 1024 * 1024 and hashlib.sha256(data).hexdigest() == ARCHIVE_SHA
        archive.write_bytes(data)
    assert digest(archive) == ARCHIVE_SHA, 'The locked musl archive differs'
    output = ROOT / 'build/IndependentKernel' / f'Math-{time.time_ns()}'
    output.mkdir(parents=True)
    source_root = output / 'musl-1.2.5'
    with tarfile.open(archive) as bundle:
        for member in bundle.getmembers():
            destination = (output / member.name).resolve()
            assert destination.is_relative_to(source_root), member.name
            if member.isdir():
                destination.mkdir(parents=True, exist_ok=True)
            elif member.isfile():
                destination.parent.mkdir(parents=True, exist_ok=True)
                destination.write_bytes(bundle.extractfile(member).read())
            else:
                raise ValueError('Non-regular archive member: ' + member.name)
    generated = output / 'include/bits'
    generated.mkdir(parents=True)
    definitions = '\n'.join((source_root / relative).read_text() for relative in
                            ('arch/x86_64/bits/alltypes.h.in', 'include/alltypes.h.in'))
    # Exact three substitutions in upstream tools/mkalltypes.sed, without an
    # external shell/sed requirement on Windows. Preserve all other lines.
    def replace(match):
        kind, definition = match.group(1), match.group(2)
        if kind == 'TYPEDEF':
            value, name = definition.rsplit(' ', 1)
            declaration, key = f'typedef {value} {name};', name
        else:
            name, value = definition.split(' ', 1)
            declaration, key = f'{kind.lower()} {name} {value};', kind.lower() + '_' + name
        return f'#if defined(__NEED_{key}) && !defined(__DEFINED_{key})\n{declaration}\n#define __DEFINED_{key}\n#endif'
    alltypes = re.sub(r'^(TYPEDEF|STRUCT|UNION)\s+(.*);$', replace, definitions, flags=re.MULTILINE)
    (generated / 'alltypes.h').write_text(alltypes, encoding='utf-8')
    llvm = ROOT / '.tools/llvm/bin'
    compiler = llvm / 'clang.exe'
    includes = [output / 'include', source_root / 'arch/x86_64', source_root / 'arch/generic',
                source_root / 'src/include', source_root / 'src/internal', source_root / 'include']
    flags = ['--target=x86_64-unknown-none-elf', '-ffreestanding', '-fno-builtin', '-fno-stack-protector',
             '-fasynchronous-unwind-tables', '-mno-red-zone', '-mcmodel=large', '-O2',
             '-fno-fast-math', '-ffp-contract=off', '-D_XOPEN_SOURCE=700']
    for include in includes:
        flags.extend(['-I', str(include)])
    sources = sorted((source_root / 'src/math').glob('*.c'))
    sources.append(source_root / 'src/fenv/x86_64/fenv.s')
    objects = output / 'objects'
    objects.mkdir()
    def compile_one(source):
        target = objects / (source.name + '.o')
        command = [str(compiler), *flags, '-c', str(source), '-o', str(target)]
        result = subprocess.run(command, capture_output=True, text=True)
        (objects / (source.name + '.log')).write_text(result.stdout + result.stderr, encoding='utf-8')
        return source, target, command, result.returncode
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
        compiled = list(pool.map(compile_one, sources))
    failed = [str(source) for source, _, _, code in compiled if code]
    receipt = {'Kind': 'XfeExternalMathLibraryBuild', 'Passed': not failed, 'Version': '1.2.5',
               'ArchiveUrl': URL, 'ArchiveSha256': ARCHIVE_SHA, 'ContainsCosmos': False,
               'ContainsMuslOperatingSystemCode': False, 'CompilerSha256': digest(compiler),
               'SourceSha256': {str(p.relative_to(source_root)): digest(p) for p in source_root.rglob('*') if p.is_file()},
               'GeneratedHeaderSha256': digest(generated / 'alltypes.h'),
               'Commands': [command for _, _, command, _ in compiled], 'Failures': failed}
    library = output / 'libXfeExternalMath.a'
    if not failed:
        subprocess.run([str(llvm / 'llvm-ar.exe'), 'rcs', str(library),
                        *(str(target) for _, target, _, _ in compiled)], check=True)
        receipt.update(LibraryPath=str(library), LibrarySha256=digest(library))
    (output / 'BuildReceipt.json').write_text(json.dumps(receipt, indent=2), encoding='utf-8')
    print(json.dumps({'Passed': not failed, 'Failures': failed, 'Receipt': str(output / 'BuildReceipt.json'),
                      'Library': str(library)}))
    return not failed


if __name__ == '__main__':
    raise SystemExit(0 if main() else 1)