[reduce] Add a first proof-of-concept reducer implementation with sample FIRRTL dialect reducers (#1591)

- Update/rewrite the `circt-reduce` tool with a custom proof-of-concept
  reducer for the FIRRTL dialect. This is supposed to be a pathfinding
  exercise and just uses FIRRTL as an example. The intent is for other
  dialects to be able to produce sets of their own reducers that will
  then be combined by the tool to operate on some input IR.

  At this point, `circt-reduce` can be used to reduce FIRRTL test cases by
  converting as many `firrtl.module` ops into `firrtl.extmodule` ops as
  possible while still maintaining some interesting characteristic.

- Extend `circt-reduce` to support exploratively applying passes to the
  input in an effort to reduce its size. Also add the ability to specify
  an entire list of potential reduction strategies/patterns which are
  tried in order. This allows for reductions with big effect, like
  removing entire modules, to be tried first, before taking a closer
  look at individual instructions.

- Add reduction strategies to `circt-reduce` that try to replace the
  right hand side of FIRRTL connects with `invalidvalue`, and generally 
  try to remove operations if they have no results or no users of their 
  results.

- Add a reduction pattern to `circt-reduce` that replaces instances of
  `firrtl.extmodule` with a `firrtl.wire` for each port. This can
  significantly reduce the complexity of test cases by pruning the
  module hierarchy of unused modules.

- Move the `Reduction` class and sample implementations into a separate
  header and implementation file. These currently live in
  `tools/circt-reduce`, but should later move into a dedicated reduction
  framework, maybe in `include/circt/Reduce`, where dialects can easily
  access them and provide their own reduction implementations.
This commit is contained in:
Fabian Schuiki 2021-08-18 17:22:43 +02:00 committed by GitHub
parent 16d50e9fd6
commit cacbaf7210
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 527 additions and 9 deletions

View File

@ -13,6 +13,7 @@
#ifndef CIRCT_DIALECT_FIRRTL_PASSES_H
#define CIRCT_DIALECT_FIRRTL_PASSES_H
#include "mlir/Pass/PassRegistry.h"
#include "llvm/ADT/Optional.h"
#include <memory>

View File

@ -13,6 +13,7 @@ set(CIRCT_TEST_DEPENDS
circt-capi-ir-test
circt-opt
circt-translate
circt-reduce
esi-tester
handshake-runner
firtool

View File

@ -0,0 +1,3 @@
// RUN: circt-reduce --help | FileCheck %s
// CHECK: OVERVIEW: CIRCT test case reduction tool

View File

@ -0,0 +1,32 @@
// RUN: circt-reduce %s --test %S/trivial.sh --test-arg firtool | FileCheck %s
firrtl.circuit "Foo" {
// CHECK: firrtl.extmodule @FooFooFoo
firrtl.module @FooFooFoo(in %x: !firrtl.uint<1>, out %y: !firrtl.uint<1>) {
firrtl.connect %y, %x : !firrtl.uint<1>, !firrtl.uint<1>
}
// CHECK: firrtl.extmodule @FooFooBar
firrtl.module @FooFooBar(in %x: !firrtl.uint<1>, out %y: !firrtl.uint<1>) {
firrtl.connect %y, %x : !firrtl.uint<1>, !firrtl.uint<1>
}
// CHECK: firrtl.module @FooFoo
firrtl.module @FooFoo(in %x: !firrtl.uint<1>, out %y: !firrtl.uint<1>) {
%x0_x, %x0_y = firrtl.instance @FooFooFoo {name = "x0"} : !firrtl.uint<1>, !firrtl.uint<1>
%x1_x, %x1_y = firrtl.instance @FooFooBar {name = "x1"} : !firrtl.uint<1>, !firrtl.uint<1>
firrtl.connect %x0_x, %x : !firrtl.uint<1>, !firrtl.uint<1>
// Skip %x1_x to trigger a "sink not fully initialized" warning
firrtl.connect %y, %x0_y : !firrtl.uint<1>, !firrtl.uint<1>
}
// CHECK: firrtl.extmodule @FooBar
firrtl.module @FooBar(in %x: !firrtl.uint<1>, out %y: !firrtl.uint<1>) {
firrtl.connect %y, %x : !firrtl.uint<1>, !firrtl.uint<1>
}
// CHECK: firrtl.extmodule @Foo
firrtl.module @Foo(in %x: !firrtl.uint<1>, out %y: !firrtl.uint<1>) {
%x0_x, %x0_y = firrtl.instance @FooFoo {name = "x0"} : !firrtl.uint<1>, !firrtl.uint<1>
%x1_x, %x1_y = firrtl.instance @FooBar {name = "x1"} : !firrtl.uint<1>, !firrtl.uint<1>
firrtl.connect %x0_x, %x : !firrtl.uint<1>, !firrtl.uint<1>
firrtl.connect %x1_x, %x : !firrtl.uint<1>, !firrtl.uint<1>
firrtl.connect %y, %x0_y : !firrtl.uint<1>, !firrtl.uint<1>
}
}

2
test/circt-reduce/trivial.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/sh
! "$1" "$2" 2>&1 | grep "error: sink \"x1.x\" not fully initialized" >/dev/null

View File

@ -56,8 +56,8 @@ tool_dirs = [
config.circt_tools_dir, config.mlir_tools_dir, config.llvm_tools_dir
]
tools = [
'firtool', 'handshake-runner', 'circt-opt', 'circt-translate',
'circt-capi-ir-test', 'esi-tester'
'firtool', 'handshake-runner', 'circt-opt', 'circt-reduce',
'circt-translate', 'circt-capi-ir-test', 'esi-tester'
]
# Enable Verilator if it has been detected.

View File

@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
add_llvm_tool(circt-reduce
circt-reduce.cpp
Reduction.cpp
)
llvm_update_compile_flags(circt-reduce)
target_link_libraries(circt-reduce
@ -11,6 +12,7 @@ target_link_libraries(circt-reduce
CIRCTCalyx
CIRCTESI
CIRCTFIRRTL
CIRCTFIRRTLTransforms
CIRCTHandshakeOps
CIRCTLLHD
CIRCTMSFT

View File

@ -0,0 +1,203 @@
//===- Reduction.cpp - Reductions for circt-reduce ------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines abstract reduction patterns for the 'circt-reduce' tool.
//
//===----------------------------------------------------------------------===//
#include "circt/Dialect/FIRRTL/FIRRTLOps.h"
#include "circt/Dialect/FIRRTL/Passes.h"
#include "circt/InitAllDialects.h"
#include "mlir/IR/AsmState.h"
#include "mlir/IR/ImplicitLocOpBuilder.h"
#include "mlir/Parser.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Pass/PassRegistry.h"
#include "mlir/Reducer/Tester.h"
#include "mlir/Support/FileUtilities.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/Passes.h"
#include "llvm/Support/Debug.h"
#include "Reduction.h"
#define DEBUG_TYPE "circt-reduce"
using namespace llvm;
using namespace mlir;
using namespace circt;
//===----------------------------------------------------------------------===//
// Reduction
//===----------------------------------------------------------------------===//
Reduction::~Reduction() {}
//===----------------------------------------------------------------------===//
// Pass Reduction
//===----------------------------------------------------------------------===//
PassReduction::PassReduction(MLIRContext *context, std::unique_ptr<Pass> pass,
bool canIncreaseSize)
: context(context), canIncreaseSize(canIncreaseSize) {
passName = pass->getArgument();
if (passName.empty())
passName = pass->getName();
if (auto opName = pass->getOpName())
pm = std::make_unique<PassManager>(context, *opName);
else
pm = std::make_unique<PassManager>(context);
pm->addPass(std::move(pass));
}
bool PassReduction::match(Operation *op) const {
return op->getName().getIdentifier() == pm->getOpName(*context);
}
LogicalResult PassReduction::rewrite(Operation *op) const {
return pm->run(op);
}
std::string PassReduction::getName() const { return passName.str(); }
//===----------------------------------------------------------------------===//
// Concrete Sample Reductions (to later move into the dialects)
//===----------------------------------------------------------------------===//
/// A sample reduction pattern that maps `firrtl.module` to `firrtl.extmodule`.
struct ModuleExternalizer : public Reduction {
bool match(Operation *op) const override {
return isa<firrtl::FModuleOp>(op);
}
LogicalResult rewrite(Operation *op) const override {
auto module = cast<firrtl::FModuleOp>(op);
OpBuilder builder(module);
builder.create<firrtl::FExtModuleOp>(
module->getLoc(),
module->getAttrOfType<StringAttr>(SymbolTable::getSymbolAttrName()),
firrtl::getModulePortInfo(module), StringRef(),
module.annotationsAttr());
module->erase();
return success();
}
std::string getName() const override { return "module-externalizer"; }
};
/// Starting at the given `op`, traverse through it and its operands and erase
/// operations that have no more uses.
static void pruneUnusedOps(Operation *initialOp) {
SmallVector<Operation *> worklist;
worklist.push_back(initialOp);
while (!worklist.empty()) {
auto op = worklist.pop_back_val();
if (!op->use_empty())
continue;
for (auto arg : op->getOperands())
if (auto argOp = arg.getDefiningOp())
worklist.push_back(argOp);
op->erase();
}
}
/// A sample reduction pattern that replaces the right-hand-side of
/// `firrtl.connect` and `firrtl.partialconnect` operations with a
/// `firrtl.invalidvalue`. This removes uses from the fanin cone to these
/// connects and creates opportunities for reduction in DCE/CSE.
struct ConnectInvalidator : public Reduction {
bool match(Operation *op) const override {
return isa<firrtl::ConnectOp, firrtl::PartialConnectOp>(op) &&
!op->getOperand(1).getDefiningOp<firrtl::InvalidValueOp>();
}
LogicalResult rewrite(Operation *op) const override {
assert(match(op));
auto rhs = op->getOperand(1);
OpBuilder builder(op);
auto invOp =
builder.create<firrtl::InvalidValueOp>(rhs.getLoc(), rhs.getType());
op->setOperand(1, invOp);
if (auto rhsOp = rhs.getDefiningOp())
pruneUnusedOps(rhsOp);
return success();
}
std::string getName() const override { return "connect-invalidator"; }
};
/// A sample reduction pattern that removes operations which either produce no
/// results or their results have no users.
struct OperationPruner : public Reduction {
bool match(Operation *op) const override {
return !isa<ModuleOp>(op) &&
!op->hasAttr(SymbolTable::getSymbolAttrName()) &&
(op->getNumResults() == 0 || op->use_empty());
}
LogicalResult rewrite(Operation *op) const override {
assert(match(op));
pruneUnusedOps(op);
return success();
}
std::string getName() const override { return "operation-pruner"; }
};
/// A sample reduction pattern that replaces instances of `firrtl.extmodule`
/// with wires.
struct ExtmoduleInstanceRemover : public Reduction {
bool match(Operation *op) const override {
if (auto instOp = dyn_cast<firrtl::InstanceOp>(op))
return isa<firrtl::FExtModuleOp>(instOp.getReferencedModule());
return false;
}
LogicalResult rewrite(Operation *op) const override {
auto instOp = cast<firrtl::InstanceOp>(op);
auto portInfo = firrtl::getModulePortInfo(instOp.getReferencedModule());
ImplicitLocOpBuilder builder(instOp.getLoc(), instOp);
SmallVector<Value> replacementWires;
for (firrtl::ModulePortInfo info : portInfo) {
auto wire = builder.create<firrtl::WireOp>(
info.type, (Twine(instOp.name()) + "_" + info.getName()).str());
if (info.isOutput()) {
auto inv = builder.create<firrtl::InvalidValueOp>(info.type);
builder.create<firrtl::ConnectOp>(wire, inv);
}
replacementWires.push_back(wire);
}
instOp.replaceAllUsesWith(std::move(replacementWires));
instOp->erase();
return success();
}
std::string getName() const override { return "extmodule-instance-remover"; }
bool acceptSizeIncrease() const override { return true; }
};
//===----------------------------------------------------------------------===//
// Reduction Registration
//===----------------------------------------------------------------------===//
static std::unique_ptr<Pass> createSimpleCanonicalizerPass() {
GreedyRewriteConfig config;
config.useTopDownTraversal = true;
config.enableRegionSimplification = false;
return createCanonicalizerPass(config);
}
void circt::createAllReductions(
MLIRContext *context,
llvm::function_ref<void(std::unique_ptr<Reduction>)> add) {
// Gather a list of reduction patterns that we should try. Ideally these are
// sorted by decreasing reduction potential/benefit. For example, things that
// can knock out entire modules while being cheap should be tried first,
// before trying to tweak operands of individual arithmetic ops.
add(std::make_unique<ModuleExternalizer>());
add(std::make_unique<PassReduction>(context, firrtl::createInlinerPass()));
add(std::make_unique<PassReduction>(context,
createSimpleCanonicalizerPass()));
add(std::make_unique<PassReduction>(context, createCSEPass()));
add(std::make_unique<ConnectInvalidator>());
add(std::make_unique<OperationPruner>());
add(std::make_unique<ExtmoduleInstanceRemover>());
}

View File

@ -0,0 +1,81 @@
//===- Reduction.h - Reductions for circt-reduce --------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines abstract reduction patterns for the 'circt-reduce' tool.
//
//===----------------------------------------------------------------------===//
#ifndef CIRCT_REDUCE_REDUCTION_H
#define CIRCT_REDUCE_REDUCTION_H
#include <memory>
#include <string>
namespace llvm {
class StringRef;
template <typename T>
class function_ref;
} // namespace llvm
namespace mlir {
struct LogicalResult;
class MLIRContext;
class Operation;
class Pass;
class PassManager;
} // namespace mlir
namespace circt {
/// An abstract reduction pattern.
struct Reduction {
virtual ~Reduction();
/// Check if the reduction can apply to a specific operation.
virtual bool match(mlir::Operation *op) const = 0;
/// Apply the reduction to a specific operation. If the returned result
/// indicates that the application failed, the resulting module is treated the
/// same as if the tester marked it as uninteresting.
virtual mlir::LogicalResult rewrite(mlir::Operation *op) const = 0;
/// Return a human-readable name for this reduction pattern.
virtual std::string getName() const = 0;
/// Return true if the tool should accept the transformation this reduction
/// performs on the module even if the overall size of the output increases.
/// This can be handy for patterns that reduce the complexity of the IR at the
/// cost of some verbosity.
virtual bool acceptSizeIncrease() const { return false; }
};
/// A reduction pattern that applies an `mlir::Pass`.
struct PassReduction : public Reduction {
PassReduction(mlir::MLIRContext *context, std::unique_ptr<mlir::Pass> pass,
bool canIncreaseSize = false);
bool match(mlir::Operation *op) const override;
mlir::LogicalResult rewrite(mlir::Operation *op) const override;
std::string getName() const override;
bool acceptSizeIncrease() const override { return canIncreaseSize; }
protected:
mlir::MLIRContext *const context;
std::unique_ptr<mlir::PassManager> pm;
llvm::StringRef passName;
bool canIncreaseSize;
};
/// Calls the function `add` with each available reduction, in the order they
/// should be applied.
void createAllReductions(
mlir::MLIRContext *context,
llvm::function_ref<void(std::unique_ptr<Reduction>)> add);
} // namespace circt
#endif // CIRCT_REDUCE_REDUCTION_H

View File

@ -12,12 +12,205 @@
//===----------------------------------------------------------------------===//
#include "circt/InitAllDialects.h"
#include "mlir/Pass/PassRegistry.h"
#include "mlir/Tools/mlir-reduce/MlirReduceMain.h"
#include "mlir/IR/AsmState.h"
#include "mlir/Parser.h"
#include "mlir/Reducer/Tester.h"
#include "mlir/Support/FileUtilities.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/ToolOutputFile.h"
int main(int argc, char **argv) {
mlir::DialectRegistry registry;
circt::registerAllDialects(registry);
mlir::MLIRContext context(registry);
return failed(mlirReduceMain(argc, argv, context));
#include "Reduction.h"
#define DEBUG_TYPE "circt-reduce"
using namespace llvm;
using namespace mlir;
using namespace circt;
//===----------------------------------------------------------------------===//
// Options
//===----------------------------------------------------------------------===//
static cl::opt<std::string>
inputFilename(cl::Positional, cl::desc("<input file>"), cl::Required);
static cl::opt<std::string>
outputFilename("o", cl::init("-"),
cl::desc("Output filename for the reduced test case"));
static cl::opt<bool>
keepBest("keep-best", cl::init(false),
cl::desc("Keep overwriting the output with better reductions"));
static cl::opt<std::string> testerCommand(
"test", cl::Required,
cl::desc("A command or script to check if output is interesting"));
static cl::list<std::string>
testerArgs("test-arg", cl::ZeroOrMore,
cl::desc("Additional arguments to the test"));
//===----------------------------------------------------------------------===//
// Tool Implementation
//===----------------------------------------------------------------------===//
/// Helper function that writes the current MLIR module to the configured output
/// file. Called for intermediate states if the `keepBest` options has been set,
/// or at least at the very end of the run.
static LogicalResult writeOutput(ModuleOp module) {
std::string errorMessage;
auto output = openOutputFile(outputFilename, &errorMessage);
if (!output) {
mlir::emitError(UnknownLoc::get(module.getContext()),
"unable to open output file \"")
<< outputFilename << "\": " << errorMessage << "\n";
return failure();
}
module.print(output->os());
output->keep();
return success();
}
/// Execute the main chunk of work of the tool. This function reads the input
/// module and iteratively applies the reduction strategies until no options
/// make it smaller.
static LogicalResult execute(MLIRContext &context) {
std::string errorMessage;
// Parse the input file.
LLVM_DEBUG(llvm::dbgs() << "Reading input\n");
OwningModuleRef module = parseSourceFile(inputFilename, &context);
if (!module)
return failure();
// Evaluate the unreduced input.
LLVM_DEBUG({
llvm::dbgs() << "Testing input with `" << testerCommand << "`\n";
for (auto &arg : testerArgs)
llvm::dbgs() << " with argument `" << arg << "`\n";
});
Tester tester(testerCommand, testerArgs);
auto initialTest = tester.isInteresting(module.get());
if (initialTest.first != Tester::Interestingness::True) {
mlir::emitError(UnknownLoc::get(&context), "input is not interesting");
return failure();
}
auto bestSize = initialTest.second;
LLVM_DEBUG(llvm::dbgs() << "Initial module has size " << bestSize << "\n");
// Gather a list of reduction patterns that we should try.
SmallVector<std::unique_ptr<Reduction>> patterns;
createAllReductions(&context, [&](auto reduction) {
patterns.push_back(std::move(reduction));
});
// Iteratively reduce the input module by applying the current reduction
// pattern to successively smaller subsets of the operations until we find one
// that retains the interesting behavior.
// ModuleExternalizer pattern;
for (unsigned patternIdx = 0; patternIdx < patterns.size();) {
Reduction &pattern = *patterns[patternIdx];
LLVM_DEBUG(llvm::dbgs()
<< "Trying reduction `" << pattern.getName() << "`\n");
size_t rangeBase = 0;
size_t rangeLength = -1;
bool patternDidReduce = false;
while (rangeLength > 0) {
// Apply the pattern to the subset of operations selected by `rangeBase`
// and `rangeLength`.
size_t opIdx = 0;
OwningModuleRef newModule = module->clone();
newModule->walk([&](Operation *op) {
if (!pattern.match(op))
return;
auto i = opIdx++;
if (i < rangeBase || i - rangeBase >= rangeLength)
return;
(void)pattern.rewrite(op);
});
if (opIdx == 0) {
LLVM_DEBUG(llvm::dbgs() << "- No more ops where the pattern applies\n");
break;
}
// Check if this reduced module is still interesting, and its overall size
// is smaller than what we had before.
auto test = tester.isInteresting(newModule.get());
if (test.first == Tester::Interestingness::True &&
(test.second < bestSize || pattern.acceptSizeIncrease())) {
// Make this reduced module the new baseline and reset our search
// strategy to start again from the beginning, since this reduction may
// have created additional opportunities.
patternDidReduce = true;
bestSize = test.second;
LLVM_DEBUG(llvm::dbgs()
<< "- Accepting module of size " << bestSize << "\n");
module = std::move(newModule);
// If this was already a run across all operations, no need to restart
// again at the top. We're done at this point.
if (rangeLength == (size_t)-1) {
rangeLength = 0;
} else {
rangeBase = 0;
rangeLength = -1;
}
// Write the current state to disk if the user asked for it.
if (keepBest)
if (failed(writeOutput(module.get())))
return failure();
} else {
// Try the pattern on the next `rangeLength` number of operations. If we
// go past the end of the input, reduce the size of the chunk of
// operations we're reducing and start again from the top.
rangeBase += rangeLength;
if (rangeBase >= opIdx) {
// Exhausted all subsets of this size. Try to go smaller.
rangeLength = std::min(rangeLength, opIdx) / 2;
rangeBase = 0;
if (rangeLength > 0)
LLVM_DEBUG(llvm::dbgs()
<< "- Trying " << rangeLength << " ops at once\n");
}
}
}
// If the pattern provided a successful reduction, restart with the first
// pattern again, since we might have uncovered additional reduction
// opportunities. Otherwise we just keep going to try the next pattern.
if (patternDidReduce && patternIdx > 0) {
LLVM_DEBUG(llvm::dbgs() << "- Reduction `" << pattern.getName()
<< "` was successful, starting at the top\n\n");
patternIdx = 0;
} else {
++patternIdx;
}
}
// Write the reduced test case to the output.
LLVM_DEBUG(llvm::dbgs() << "All reduction strategies exhausted\n");
return writeOutput(module.get());
}
/// The entry point for the `circt-reduce` tool. Configures and parses the
/// command line options, registers all dialects with a context, and calls the
/// `execute` function to do the actual work.
int main(int argc, char **argv) {
llvm::InitLLVM y(argc, argv);
// Parse the command line options provided by the user.
registerMLIRContextCLOptions();
registerAsmPrinterCLOptions();
cl::ParseCommandLineOptions(argc, argv, "CIRCT test case reduction tool\n");
// Register all the dialects and create a context to work wtih.
mlir::DialectRegistry registry;
registerAllDialects(registry);
mlir::MLIRContext context(registry);
// Do the actual processing and use `exit` to avoid the slow teardown of the
// context.
exit(failed(execute(context)));
}