cxxrtl: pull in CompilationUnit etc. from Chryse.

niar/build.py

@@ -1,4 +1,3 @@
-import importlib
 import inspect
 import logging
 import re
@@ -7,7 +6,7 @@ from typing import Optional
 
 from amaranth.build import Platform
 
-from .logger import logger, logtime
+from .logging import logger, logtime
 from .project import Project
 
 __all__ = ["add_arguments"]

niar/cmdrunner.py

@@ -0,0 +1,145 @@
+import hashlib
+import inspect
+import shlex
+import subprocess
+
+from .logging import logger
+
+__all__ = ["CompilationUnit", "CommandRunner"]
+
+
+class CompilationUnit:
+    def __init__(self, cmd, *, infs, outf, chdir):
+        if inspect.isfunction(cmd):
+            self.cmd = cmd
+        else:
+            self.cmd = [str(el) for el in cmd]
+        self.infs  = [str(p) for p in infs]
+        self.outf  = str(outf) if outf else None
+        self.chdir = chdir
+
+        self.forced = False
+        self.digest_path = f"{outf}.dig"
+
+    @property
+    def up_to_date(self):
+        if self.outf is None:
+            return False
+        try:
+            with open(self.digest_path, "r") as f:
+                return f.read() == self.digest_ins_with_cmd()
+        except FileNotFoundError:
+            return False
+
+    def mark_up_to_date(self):
+        if self.outf is None:
+            return
+        with open(self.digest_path, "w") as f:
+            f.write(self.digest_ins_with_cmd())
+
+    def digest_ins_with_cmd(self):
+        m = hashlib.sha256()
+
+        def digest_int(i):
+            m.update(f"{i:08x}".encode())
+
+        def digest_bytes(b):
+            digest_int(len(b))
+            m.update(b)
+
+        def digest_str(s):
+            digest_bytes(s.encode())
+
+        sorted_in_paths = list(self.infs)
+        sorted_in_paths.sort()
+
+        digest_int(len(sorted_in_paths))
+        for in_path in sorted_in_paths:
+            digest_str(in_path)
+            with open(in_path, "rb") as f:
+                digest_bytes(f.read())
+
+        if not inspect.isfunction(self.cmd):
+            digest_int(len(self.cmd))
+            for el in self.cmd:
+                digest_str(el)
+
+        return m.hexdigest()
+
+
+class CommandRunner:
+    cus: list[CompilationUnit]
+
+    def __init__(self, *, force=False):
+        self.cus = []
+        self.force = force
+
+    @property
+    def compile_commands(self):
+        return {cu.outf: cu.cmd for cu in self.cus}
+
+    def add_process(self, cmd, *, infs, outf, chdir=None):
+        self.cus.append(
+            CompilationUnit(cmd, infs=infs, outf=outf, chdir=chdir))
+
+    def run(self, step="compile"):
+        self.run_cus(self.cus, step)
+        self.cus = []
+
+    def run_cmd(self, cmd, *, step="compile", chdir=None):
+        cu = CompilationUnit(cmd, infs=[], outf=None, chdir=chdir)
+        self.run_cus([cu], step)
+
+    def run_cus(self, cus, step):
+        runnables = []
+        for cu in cus:
+            if cu.up_to_date:
+                if self.force:
+                    cu.forced = True
+                    runnables.append([cu, None])
+                else:
+                    self.report(cu, skip=True)
+            else:
+                runnables.append([cu, None])
+
+        for cu_proc in runnables:
+            cu = cu_proc[0]
+            self.report(cu)
+            if inspect.isfunction(cu.cmd):
+                cu.cmd()
+            else:
+                cu_proc[1] = subprocess.Popen(cu.cmd, cwd=cu.chdir)
+
+        failed = []
+        for cu, proc in runnables:
+            if proc is not None and proc.wait() != 0:
+                failed.append(cu)
+
+        if failed:
+            logger.error("the following process(es) failed:")
+            for cu in failed:
+                logger.error(f"  {formatted(cu)}")
+            raise RuntimeError(f"failed {step} step")
+
+        for cu, _ in runnables:
+            cu.mark_up_to_date()
+
+    def report(self, cu, *, skip=False):
+        if cu.forced:
+            assert(not skip)
+            action = "[force]"
+        elif skip:
+            action = "[skip] "
+        else:
+            action = "[run]  "
+        logger.info(f"{action} {formatted(cu)}")
+
+
+def formatted(cu):
+    if inspect.isfunction(cu.cmd):
+        return cu.cmd.__name__
+
+    cmd = shlex.join(cu.cmd)
+    if cu.chdir:
+        return f"(in {cu.chdir}/) {cmd}"
+    return cmd

niar/cxxrtl.py

@@ -2,19 +2,17 @@ import json
 import logging
 import os
 import shutil
-import subprocess
 from enum import Enum
 from functools import partial
 from pathlib import Path
 from typing import Any
 
-from amaranth import Elaboratable
 from amaranth._toolchain.yosys import YosysBinary, find_yosys
 from amaranth.back import rtlil
 
 from .build import construct_top
-from .cxxrtl_platform import CxxrtlPlatform
+from .cmdrunner import CommandRunner
-from .logger import logger, logtime
+from .logging import logtime
 from .project import Project
 
 __all__ = ["add_arguments"]
@@ -89,6 +87,12 @@ def add_arguments(np: Project, parser):
         type=str,
         help="output a VCD file",
     )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="don't use cached compilations",
+    )
 
 
 def main(np: Project, args):
@@ -98,24 +102,37 @@ def main(np: Project, args):
 
     platform = np.cxxrtl_target_by_name(args.target)
     design = construct_top(np, platform)
+    cr = CommandRunner(force=args.force)
+
+    with logtime(logging.DEBUG, "elaboration"):
+        il_path = np.path.build(f"{np.name}.il")
+        rtlil_text = rtlil.convert(design, name=np.name, platform=platform)
+        with open(il_path, "w") as f:
+            f.write(rtlil_text)
 
         cxxrtl_cc_path = np.path.build(f"{np.name}.cc")
-    with logtime(logging.DEBUG, "elaboration"):
+        yosys_script_path = _make_absolute(np.path.build(f"{np.name}-cxxrtl.ys"))
-        _cxxrtl_convert_with_header(
+        black_boxes = {}
-            yosys,
-            cxxrtl_cc_path,
-            design,
-            np.name,
-            platform,
-            black_boxes={},
-        )
 
-    # TODO: bring in Chryse's CompilationUnit stuff to reduce rework.
+        with open(yosys_script_path, "w") as f:
+            for box_source in black_boxes.values():
+                f.write(f"read_rtlil <<rtlil\n{box_source}\nrtlil\n")
+            f.write(f"read_rtlil {_make_absolute(il_path)}\n")
+            # TODO: do we want to call any opt passes here?
+            f.write(f"write_cxxrtl -header {_make_absolute(cxxrtl_cc_path)}\n")
 
-    cc_o_paths = {
+        def rtlil_to_cc():
-        cxxrtl_cc_path: np.path.build(f"{np.name}.o"),
+            yosys.run(["-q", yosys_script_path])
-    }
+
-    for path in np.path("cxxrtl").glob("*.cc"):
+        cr.add_process(rtlil_to_cc,
+            infs=[il_path, yosys_script_path],
+            outf=cxxrtl_cc_path)
+        cr.run()
+
+    with logtime(logging.DEBUG, "compilation"):
+        cc_o_paths = {cxxrtl_cc_path: np.path.build(f"{np.name}.o")}
+        for path in np.path("cxxrtl").glob("**/*.cc"):
+            # XXX: we make no effort to distinguish cxxrtl/a.cc and cxxrtl/dir/a.cc.
             cc_o_paths[path] = np.path.build(f"{path.stem}.o")
 
         cxxflags = CXXFLAGS + [
@@ -129,49 +146,34 @@ def main(np: Project, args):
                 "-DCXXRTL_INCLUDE_VCD_CAPI_IMPL",
             ]
 
-    procs = []
+        depfs = list(np.path("cxxrtl").glob("**/*.h"))
-    compile_commands = {}
+
         for cc_path, o_path in cc_o_paths.items():
             cmd = [
                 "c++",
                 *cxxflags,
-            "-I" + str(np.path("build")),
+                f"-I{np.path("build")}",
-            "-I"
+                f"-I{yosys.data_dir() / "include" / "backends" / "cxxrtl" / "runtime"}",
-            + str(yosys.data_dir() / "include" / "backends" / "cxxrtl" / "runtime"),
                 "-c",
-            str(cc_path),
+                cc_path,
                 "-o",
-            str(o_path),
+                o_path,
             ]
             if platform.uses_zig:
                 cmd = ["zig"] + cmd
-        compile_commands[o_path] = cmd
+            cr.add_process(cmd, infs=[cc_path] + depfs, outf=o_path)
-        logger.debug(" ".join(str(e) for e in cmd))
-        procs.append((cc_path, subprocess.Popen(cmd)))
 
         with open(np.path.build("compile_commands.json"), "w") as f:
             json.dump(
-            [
+                [{
-                {
                     "directory": str(np.path()),
-                    "file": str(file),
+                    "file": file,
                     "arguments": arguments,
-                }
+                } for file, arguments in cr.compile_commands.items()],
-                for file, arguments in compile_commands.items()
-            ],
                 f,
             )
 
-    failed = []
+        cr.run()
-    for cc_path, p in procs:
-        if p.wait() != 0:
-            failed.append(cc_path)
-
-    if failed:
-        logger.error("Failed to build paths:")
-        for p in failed:
-            logger.error(f"- {p}")
-        raise RuntimeError("failed compile step")
 
         exe_o_path = np.path.build("cxxrtl")
         if platform.uses_zig:
@@ -186,9 +188,13 @@ def main(np: Project, args):
             ]
             if args.optimize.opt_app:
                 cmd += ["-Doptimize=ReleaseFast"]
-        logger.debug(" ".join(str(e) for e in cmd))
+            outf = "cxxrtl/zig-out/bin/cxxrtl"
-        subprocess.run(cmd, cwd="cxxrtl", check=True)
+            cr.add_process(cmd,
-        shutil.copy("cxxrtl/zig-out/bin/cxxrtl", exe_o_path)
+                infs=cc_o_paths.values() + np.path("cxxrtl").glob("**/*.zig"),
+                outf=outf,
+                chdir="cxxrtl")
+            cr.run()
+            shutil.copy(outf, exe_o_path)
         else:
             cmd = [
                 "c++",
@@ -197,39 +203,22 @@ def main(np: Project, args):
                 "-o",
                 exe_o_path,
             ]
-        logger.debug(" ".join(str(e) for e in cmd))
+            cr.add_process(cmd,
-        subprocess.run(cmd, check=True)
+                infs=cc_o_paths.values(),
+                outf=exe_o_path)
+            cr.run()
 
     if not args.compile:
         cmd = [exe_o_path]
         if args.vcd:
             cmd += ["--vcd", args.vcd]
-        logger.debug(" ".join(str(e) for e in cmd))
+        cr.run_cmd(cmd, step="run")
-        subprocess.run(cmd, check=True)
 
 
-def _cxxrtl_convert_with_header(
+def _make_absolute(path):
-    yosys: YosysBinary,
+    if path.is_absolute():
-    cc_out: Path,
-    design: Elaboratable,
-    name: str,
-    platform: CxxrtlPlatform,
-    *,
-    black_boxes: dict[Any, str],
-):
-    if cc_out.is_absolute():
         try:
-            cc_out = cc_out.relative_to(Path.cwd())
+            path = path.relative_to(Path.cwd())
         except ValueError:
-            raise AssertionError(
+            raise AssertionError("path must be relative to cwd for builtin-yosys to access it")
-                "cc_out must be relative to cwd for builtin-yosys to write to it"
+    return path
-            )
-    rtlil_out = f"{cc_out}.il"
-    rtlil_text = rtlil.convert(design, name=name, platform=platform)
-    script = []
-    for box_source in black_boxes.values():
-        script.append(f"read_rtlil <<rtlil\n{box_source}\nrtlil")
-    script.append(f"read_rtlil <<rtlil\n{rtlil_text}\nrtlil")
-    script.append(f"write_rtlil {rtlil_out}")
-    script.append(f"write_cxxrtl -header {cc_out}")
-    yosys.run(["-q", "-"], "\n".join(script))

niar/test_cli.py

@@ -4,7 +4,7 @@ from argparse import ArgumentParser
 from amaranth import Elaboratable, Module
 from amaranth_boards.icebreaker import ICEBreakerPlatform
 
-from . import Project, build, logger
+from . import Project, build, logging
 
 calling_test = False
 
@@ -22,8 +22,8 @@ class FixtureProject(Project):
 
 class TestCLI(unittest.TestCase):
     def setUp(self):
-        logger.disable()
+        logging.disable()
-        self.addCleanup(logger.enable)
+        self.addCleanup(logging.enable)
 
     def test_build_works(self):
         parser = ArgumentParser()