Add unit test discovery and execution tool (#7685)

Add the `circt-test` tool. This tool is intended to be a driver for
discovering and executing hardware unit tests in an MLIR input file. In
this first draft circt-test simply parses an MLIR assembly or bytecode
file and prints out a list of `verif.formal` operations.

This is just a starting point. We'll want this tool to be able to also
generate the Verilog code for one or more of the listed unit tests, run
the tests through tools and collect results, and much more. From a user
perspective, calling something like `circt-test design.mlirbc` should
do a sane default run of all unit tests in the provided input. But the
tool should also be useful for build systems to discover tests and run
them individually.

test/CMakeLists.txt

@@ -26,6 +26,7 @@ set(CIRCT_TEST_DEPENDS
   circt-dis
   circt-lec
   circt-opt
+  circt-test
   circt-translate
   circt-reduce
   handshake-runner

test/circt-test/basic.mlir

@@ -0,0 +1,43 @@
+// RUN: circt-test %s | FileCheck %s
+// RUN: circt-test %s --json | FileCheck --check-prefix=JSON %s
+// RUN: circt-as %s -o - | circt-test | FileCheck %s
+
+// JSON: [
+
+// JSON-NEXT: {
+// JSON-NEXT:   "name": "Some.TestA"
+// JSON-NEXT:   "kind": "formal"
+// JSON-NEXT: }
+// JSON-NEXT: {
+// JSON-NEXT:   "name": "Some.TestB"
+// JSON-NEXT:   "kind": "formal"
+// JSON-NEXT: }
+// CHECK: Some.TestA formal {}
+// CHECK: Some.TestB formal {}
+verif.formal @Some.TestA (k=42) {}
+verif.formal @Some.TestB (k=42) {}
+
+// JSON-NEXT: {
+// JSON-NEXT:   "name": "Attrs"
+// JSON-NEXT:   "kind": "formal"
+// JSON-NEXT:   "attrs": {
+// JSON-NEXT:     "awesome": true
+// JSON-NEXT:     "engine": "bmc"
+// JSON-NEXT:     "offset": 42
+// JSON-NEXT:     "tags": [
+// JSON-NEXT:       "sby"
+// JSON-NEXT:       "induction"
+// JSON-NEXT:     ]
+// JSON-NEXT:     "wow": false
+// JSON-NEXT:   }
+// JSON-NEXT: }
+// CHECK: Attrs formal {awesome = true, engine = "bmc", offset = 42 : i64, tags = ["sby", "induction"], wow = false}
+verif.formal @Attrs (k=42) attributes {
+    awesome = true,
+    engine = "bmc",
+    offset = 42 : i64,
+    tags = ["sby", "induction"],
+    wow = false
+} {}
+
+// JSON: ]

test/circt-test/commandline.mlir

@@ -0,0 +1,3 @@
+// RUN: circt-test --help | FileCheck %s
+
+// CHECK: OVERVIEW: Hardware unit testing tool

test/lit.cfg.py

@@ -60,8 +60,8 @@ tool_dirs = [
 tools = [
     'arcilator', 'circt-as', 'circt-capi-ir-test', 'circt-capi-om-test',
     'circt-capi-firrtl-test', 'circt-capi-firtool-test', 'circt-dis',
-    'circt-lec', 'circt-reduce', 'circt-translate', 'firtool', 'hlstool',
-    'om-linker', 'ibistool'
+    'circt-lec', 'circt-reduce', 'circt-test', 'circt-translate', 'firtool',
+    'hlstool', 'om-linker', 'ibistool'
 ]
 
 if "CIRCT_OPT_CHECK_IR_ROUNDTRIP" in os.environ:

tools/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory(circt-lsp-server)
 add_subdirectory(circt-opt)
 add_subdirectory(circt-reduce)
 add_subdirectory(circt-rtl-sim)
+add_subdirectory(circt-test)
 add_subdirectory(circt-translate)
 add_subdirectory(firtool)
 add_subdirectory(handshake-runner)

tools/circt-test/CMakeLists.txt

@@ -0,0 +1,26 @@
+set(libs
+  CIRCTComb
+  CIRCTHW
+  CIRCTOM
+  CIRCTSeq
+  CIRCTSim
+  CIRCTSV
+  CIRCTVerif
+
+  MLIRLLVMDialect
+  MLIRArithDialect
+  MLIRControlFlowDialect
+  MLIRFuncDialect
+  MLIRSCFDialect
+
+  MLIRBytecodeReader
+  MLIRIR
+  MLIRParser
+  MLIRSupport
+)
+
+add_circt_tool(circt-test circt-test.cpp DEPENDS ${libs})
+target_link_libraries(circt-test PRIVATE ${libs})
+
+llvm_update_compile_flags(circt-test)
+mlir_check_all_link_libraries(circt-test)

tools/circt-test/circt-test.cpp

@@ -0,0 +1,167 @@
+//===- circt-test.cpp - Hardware unit test discovery and execution tool ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Discover runnable unit tests in an MLIR blob and execute them through various
+// backends.
+//
+//===----------------------------------------------------------------------===//
+
+#include "circt/Dialect/Comb/CombDialect.h"
+#include "circt/Dialect/HW/HWDialect.h"
+#include "circt/Dialect/OM/OMDialect.h"
+#include "circt/Dialect/SV/SVDialect.h"
+#include "circt/Dialect/Seq/SeqDialect.h"
+#include "circt/Dialect/Sim/SimDialect.h"
+#include "circt/Dialect/Verif/VerifDialect.h"
+#include "circt/Dialect/Verif/VerifOps.h"
+#include "circt/Support/JSON.h"
+#include "circt/Support/Version.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Parser/Parser.h"
+#include "mlir/Support/FileUtilities.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/WithColor.h"
+
+#include <string>
+
+using namespace llvm;
+using namespace mlir;
+using namespace circt;
+
+//===----------------------------------------------------------------------===//
+// Command Line Options
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+/// The tool's command line options.
+struct Options {
+  cl::OptionCategory cat{"circt-test Options"};
+  cl::opt<std::string> inputFilename{cl::Positional, cl::desc("<input file>"),
+                                     cl::init("-"), cl::cat(cat)};
+  cl::opt<std::string> outputFilename{
+      "o", cl::desc("Output filename (`-` for stdout)"),
+      cl::value_desc("filename"), cl::init("-"), cl::cat(cat)};
+  cl::opt<bool> json{"json", cl::desc("Emit test list as JSON array"),
+                     cl::init(false), cl::cat(cat)};
+};
+Options opts;
+
+} // namespace
+
+//===----------------------------------------------------------------------===//
+// Tool Implementation
+//===----------------------------------------------------------------------===//
+
+/// List all the tests in a given module.
+static LogicalResult listTests(ModuleOp module, llvm::raw_ostream &output) {
+  // Handle JSON output.
+  if (opts.json) {
+    json::OStream json(output, 2);
+    json.arrayBegin();
+    auto result = module.walk([&](Operation *op) {
+      if (auto formalOp = dyn_cast<verif::FormalOp>(op)) {
+        json.objectBegin();
+        auto guard = make_scope_exit([&] { json.objectEnd(); });
+        json.attribute("name", formalOp.getSymName());
+        json.attribute("kind", "formal");
+        auto attrs = formalOp->getDiscardableAttrDictionary();
+        if (!attrs.empty()) {
+          json.attributeBegin("attrs");
+          auto guard = make_scope_exit([&] { json.attributeEnd(); });
+          if (failed(convertAttributeToJSON(json, attrs))) {
+            op->emitError() << "unsupported attributes: `" << attrs
+                            << "` cannot be converted to JSON";
+            return WalkResult::interrupt();
+          }
+        }
+      }
+      return WalkResult::advance();
+    });
+    json.arrayEnd();
+    return failure(result.wasInterrupted());
+  }
+
+  // Handle regular text output.
+  module.walk([&](Operation *op) {
+    if (auto formalOp = dyn_cast<verif::FormalOp>(op)) {
+      output << formalOp.getSymName() << "  formal"
+             << "  " << formalOp->getDiscardableAttrDictionary() << "\n";
+    }
+  });
+  return success();
+}
+
+/// Entry point for the circt-test tool. At this point an MLIRContext is
+/// available, all dialects have been registered, and all command line options
+/// have been parsed.
+static LogicalResult execute(MLIRContext *context) {
+  SourceMgr srcMgr;
+  SourceMgrDiagnosticHandler handler(srcMgr, context);
+
+  // Open the output file for writing.
+  std::string errorMessage;
+  auto output = openOutputFile(opts.outputFilename, &errorMessage);
+  if (!output)
+    return emitError(UnknownLoc::get(context)) << errorMessage;
+
+  // Parse the input file.
+  auto module = parseSourceFile<ModuleOp>(opts.inputFilename, srcMgr, context);
+  if (!module)
+    return failure();
+
+  // List all tests in the input.
+  if (failed(listTests(*module, output->os())))
+    return failure();
+
+  output->keep();
+  return success();
+}
+
+int main(int argc, char **argv) {
+  InitLLVM y(argc, argv);
+
+  // Set the bug report message to indicate users should file issues on
+  // llvm/circt and not llvm/llvm-project.
+  setBugReportMsg(circtBugReportMsg);
+
+  // Print the CIRCT version when requested.
+  cl::AddExtraVersionPrinter(
+      [](raw_ostream &os) { os << getCirctVersion() << '\n'; });
+
+  // Register the dialects.
+  DialectRegistry registry;
+  registry.insert<circt::comb::CombDialect>();
+  registry.insert<circt::hw::HWDialect>();
+  registry.insert<circt::om::OMDialect>();
+  registry.insert<circt::seq::SeqDialect>();
+  registry.insert<circt::sim::SimDialect>();
+  registry.insert<circt::sv::SVDialect>();
+  registry.insert<circt::verif::VerifDialect>();
+  registry.insert<mlir::LLVM::LLVMDialect>();
+  registry.insert<mlir::func::FuncDialect>();
+  registry.insert<mlir::arith::ArithDialect>();
+  registry.insert<mlir::cf::ControlFlowDialect>();
+  registry.insert<mlir::scf::SCFDialect>();
+
+  // Hide default LLVM options, other than for this tool.
+  // MLIR options are added below.
+  cl::HideUnrelatedOptions({&opts.cat, &llvm::getColorCategory()});
+  cl::ParseCommandLineOptions(argc, argv, "Hardware unit testing tool\n");
+
+  MLIRContext context(registry);
+  exit(failed(execute(&context)));
+}