From 46ac1c24e9f4fe49fc9f643186623d830d7e7865 Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Thu, 27 Jun 2024 13:59:43 +0300 Subject: [PATCH] cxxrtl: pull in CompilationUnit etc. from Chryse. --- niar/build.py | 3 +- niar/cmdrunner.py | 145 +++++++++++++++++++++ niar/cxxrtl.py | 223 ++++++++++++++++----------------- niar/{logger.py => logging.py} | 0 niar/test_cli.py | 6 +- 5 files changed, 255 insertions(+), 122 deletions(-) create mode 100644 niar/cmdrunner.py rename niar/{logger.py => logging.py} (100%) diff --git a/niar/build.py b/niar/build.py index e427be1..c1604e1 100644 --- a/niar/build.py +++ b/niar/build.py @@ -1,4 +1,3 @@ -import importlib import inspect import logging import re @@ -7,7 +6,7 @@ from typing import Optional from amaranth.build import Platform -from .logger import logger, logtime +from .logging import logger, logtime from .project import Project __all__ = ["add_arguments"] diff --git a/niar/cmdrunner.py b/niar/cmdrunner.py new file mode 100644 index 0000000..7004d25 --- /dev/null +++ b/niar/cmdrunner.py @@ -0,0 +1,145 @@ +import hashlib +import inspect +import shlex +import subprocess + +from .logging import logger + +__all__ = ["CompilationUnit", "CommandRunner"] + + +class CompilationUnit: + def __init__(self, cmd, *, infs, outf, chdir): + if inspect.isfunction(cmd): + self.cmd = cmd + else: + self.cmd = [str(el) for el in cmd] + self.infs = [str(p) for p in infs] + self.outf = str(outf) if outf else None + self.chdir = chdir + + self.forced = False + self.digest_path = f"{outf}.dig" + + @property + def up_to_date(self): + if self.outf is None: + return False + try: + with open(self.digest_path, "r") as f: + return f.read() == self.digest_ins_with_cmd() + except FileNotFoundError: + return False + + def mark_up_to_date(self): + if self.outf is None: + return + with open(self.digest_path, "w") as f: + f.write(self.digest_ins_with_cmd()) + + def digest_ins_with_cmd(self): + m = hashlib.sha256() + + def digest_int(i): + m.update(f"{i:08x}".encode()) + + def digest_bytes(b): + digest_int(len(b)) + m.update(b) + + def digest_str(s): + digest_bytes(s.encode()) + + sorted_in_paths = list(self.infs) + sorted_in_paths.sort() + + digest_int(len(sorted_in_paths)) + for in_path in sorted_in_paths: + digest_str(in_path) + with open(in_path, "rb") as f: + digest_bytes(f.read()) + + if not inspect.isfunction(self.cmd): + digest_int(len(self.cmd)) + for el in self.cmd: + digest_str(el) + + return m.hexdigest() + + +class CommandRunner: + cus: list[CompilationUnit] + + def __init__(self, *, force=False): + self.cus = [] + self.force = force + + @property + def compile_commands(self): + return {cu.outf: cu.cmd for cu in self.cus} + + def add_process(self, cmd, *, infs, outf, chdir=None): + self.cus.append( + CompilationUnit(cmd, infs=infs, outf=outf, chdir=chdir)) + + def run(self, step="compile"): + self.run_cus(self.cus, step) + self.cus = [] + + def run_cmd(self, cmd, *, step="compile", chdir=None): + cu = CompilationUnit(cmd, infs=[], outf=None, chdir=chdir) + self.run_cus([cu], step) + + def run_cus(self, cus, step): + runnables = [] + for cu in cus: + if cu.up_to_date: + if self.force: + cu.forced = True + runnables.append([cu, None]) + else: + self.report(cu, skip=True) + else: + runnables.append([cu, None]) + + for cu_proc in runnables: + cu = cu_proc[0] + self.report(cu) + if inspect.isfunction(cu.cmd): + cu.cmd() + else: + cu_proc[1] = subprocess.Popen(cu.cmd, cwd=cu.chdir) + + failed = [] + for cu, proc in runnables: + if proc is not None and proc.wait() != 0: + failed.append(cu) + + if failed: + logger.error("the following process(es) failed:") + for cu in failed: + logger.error(f" {formatted(cu)}") + raise RuntimeError(f"failed {step} step") + + for cu, _ in runnables: + cu.mark_up_to_date() + + def report(self, cu, *, skip=False): + if cu.forced: + assert(not skip) + action = "[force]" + elif skip: + action = "[skip] " + else: + action = "[run] " + logger.info(f"{action} {formatted(cu)}") + + +def formatted(cu): + if inspect.isfunction(cu.cmd): + return cu.cmd.__name__ + + cmd = shlex.join(cu.cmd) + if cu.chdir: + return f"(in {cu.chdir}/) {cmd}" + return cmd diff --git a/niar/cxxrtl.py b/niar/cxxrtl.py index 7c43b22..bf948c1 100644 --- a/niar/cxxrtl.py +++ b/niar/cxxrtl.py @@ -2,19 +2,17 @@ import json import logging import os import shutil -import subprocess from enum import Enum from functools import partial from pathlib import Path from typing import Any -from amaranth import Elaboratable from amaranth._toolchain.yosys import YosysBinary, find_yosys from amaranth.back import rtlil from .build import construct_top -from .cxxrtl_platform import CxxrtlPlatform -from .logger import logger, logtime +from .cmdrunner import CommandRunner +from .logging import logtime from .project import Project __all__ = ["add_arguments"] @@ -89,6 +87,12 @@ def add_arguments(np: Project, parser): type=str, help="output a VCD file", ) + parser.add_argument( + "-f", + "--force", + action="store_true", + help="don't use cached compilations", + ) def main(np: Project, args): @@ -98,138 +102,123 @@ def main(np: Project, args): platform = np.cxxrtl_target_by_name(args.target) design = construct_top(np, platform) + cr = CommandRunner(force=args.force) - cxxrtl_cc_path = np.path.build(f"{np.name}.cc") with logtime(logging.DEBUG, "elaboration"): - _cxxrtl_convert_with_header( - yosys, - cxxrtl_cc_path, - design, - np.name, - platform, - black_boxes={}, - ) + il_path = np.path.build(f"{np.name}.il") + rtlil_text = rtlil.convert(design, name=np.name, platform=platform) + with open(il_path, "w") as f: + f.write(rtlil_text) - # TODO: bring in Chryse's CompilationUnit stuff to reduce rework. + cxxrtl_cc_path = np.path.build(f"{np.name}.cc") + yosys_script_path = _make_absolute(np.path.build(f"{np.name}-cxxrtl.ys")) + black_boxes = {} - cc_o_paths = { - cxxrtl_cc_path: np.path.build(f"{np.name}.o"), - } - for path in np.path("cxxrtl").glob("*.cc"): - cc_o_paths[path] = np.path.build(f"{path.stem}.o") + with open(yosys_script_path, "w") as f: + for box_source in black_boxes.values(): + f.write(f"read_rtlil <