[mlir] Add an out-of-tree dialect example

This adds a minimal out-of-tree dialect template which can be used to start work on a standalone dialect implementation without having to integrate it in the main LLVM tree.

It mostly sets up the directory structure and provides CMakeLists.txt files to build a dialect library, an opt-like tool to operate on that dialect as well as tests. It could be expanded in the future to add examples of more user-defined operations, types, attributes, generated enums, transforms, etc. and linked to a tutorial.

Differential Revision: https://reviews.llvm.org/D77133

mlir/examples/standalone/CMakeLists.txt

@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 3.10)
+
+if(POLICY CMP0068)
+  cmake_policy(SET CMP0068 NEW)
+  set(CMAKE_BUILD_WITH_INSTALL_NAME_DIR ON)
+endif()
+
+if(POLICY CMP0075)
+  cmake_policy(SET CMP0075 NEW)
+endif()
+
+if(POLICY CMP0077)
+  cmake_policy(SET CMP0077 NEW)
+endif()
+
+project(standalone-dialect LANGUAGES CXX C)
+
+find_package(MLIR REQUIRED CONFIG)
+
+message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}")
+message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+
+set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin)
+set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib)
+set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR})
+
+list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}")
+list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+include(TableGen)
+include(AddLLVM)
+include(AddMLIR)
+include(HandleLLVMOptions)
+
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(${MLIR_INCLUDE_DIRS})
+include_directories(${PROJECT_SOURCE_DIR}/include)
+include_directories(${PROJECT_BINARY_DIR}/include)
+link_directories(${LLVM_BUILD_LIBRARY_DIR})
+add_definitions(${LLVM_DEFINITIONS})
+
+add_subdirectory(include)
+add_subdirectory(lib)
+add_subdirectory(test)
+add_subdirectory(standalone-opt)

mlir/examples/standalone/README.md

@@ -0,0 +1,18 @@
+# An out-of-tree MLIR dialect
+
+This is an example of an out-of-tree [MLIR](https://mlir.llvm.org/) dialect along with a standalone `opt`-like tool to operate on that dialect.
+
+## Building
+
+This setup assumes that you have built LLVM and MLIR in `$BUILD_DIR` and installed them to `$PREFIX`. To build and launch the tests, run
+```sh
+mkdir build && cd build
+cmake -G Ninja .. -DMLIR_DIR=$PREFIX/lib/cmake/mlir -DLLVM_EXTERNAL_LIT=$BUILD_DIR/bin/llvm-lit
+cmake --build . --target check-standalone
+```
+To build the documentation from the TableGen description of the dialect operations, run
+```sh
+cmake --build . --target mlir-doc
+```
+**Note**: Make sure to pass `-DLLVM_INSTALL_UTILS=ON` when building LLVM with CMake in order to install `FileCheck` to the chosen installation prefix.
+

mlir/examples/standalone/include/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(Standalone)

mlir/examples/standalone/include/Standalone/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_mlir_dialect(StandaloneOps standalone)
+add_mlir_doc(StandaloneDialect -gen-dialect-doc StandaloneDialect Standalone/)
+add_mlir_doc(StandaloneOps -gen-op-doc StandaloneOps Standalone/)

mlir/examples/standalone/include/Standalone/StandaloneDialect.h

@@ -0,0 +1,22 @@
+//===- StandaloneDialect.h - Standalone dialect -----------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef STANDALONE_STANDALONEDIALECT_H
+#define STANDALONE_STANDALONEDIALECT_H
+
+#include "mlir/IR/Dialect.h"
+
+namespace mlir {
+namespace standalone {
+
+#include "Standalone/StandaloneOpsDialect.h.inc"
+
+} // namespace standalone
+} // namespace mlir
+
+#endif // STANDALONE_STANDALONEDIALECT_H

mlir/examples/standalone/include/Standalone/StandaloneDialect.td

@@ -0,0 +1,36 @@
+//===- StandaloneDialect.td - Standalone dialect -----------*- tablegen -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef STANDALONE_DIALECT
+#define STANDALONE_DIALECT
+
+include "mlir/IR/OpBase.td"
+
+//===----------------------------------------------------------------------===//
+// Standalone dialect definition.
+//===----------------------------------------------------------------------===//
+
+def Standalone_Dialect : Dialect {
+    let name = "standalone";
+    let summary = "A standalone out-of-tree MLIR dialect.";
+    let description = [{
+        This dialect is an example of an out-of-tree MLIR dialect designed to
+        illustrate the basic setup required to develop MLIR-based tools without
+        working inside of the LLVM source tree.
+    }];
+    let cppNamespace = "standalone";
+}
+
+//===----------------------------------------------------------------------===//
+// Base standalone operation definition.
+//===----------------------------------------------------------------------===//
+
+class Standalone_Op<string mnemonic, list<OpTrait> traits = []> :
+        Op<Standalone_Dialect, mnemonic, traits>;
+
+#endif // STANDALONE_DIALECT

mlir/examples/standalone/include/Standalone/StandaloneOps.h

@@ -0,0 +1,25 @@
+//===- StandaloneOps.h - Standalone dialect ops -----------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef STANDALONE_STANDALONEOPS_H
+#define STANDALONE_STANDALONEOPS_H
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Interfaces/SideEffects.h"
+
+namespace mlir {
+namespace standalone {
+
+#define GET_OP_CLASSES
+#include "Standalone/StandaloneOps.h.inc"
+
+} // namespace standalone
+} // namespace mlir
+
+#endif // STANDALONE_STANDALONEOPS_H

mlir/examples/standalone/include/Standalone/StandaloneOps.td

@@ -0,0 +1,42 @@
+//===- StandaloneOps.td - Standalone dialect ops -----------*- tablegen -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef STANDALONE_OPS
+#define STANDALONE_OPS
+
+include "StandaloneDialect.td"
+include "mlir/Interfaces/SideEffects.td"
+
+def Standalone_FooOp : Standalone_Op<"foo", [NoSideEffect,
+                                             SameOperandsAndResultType]> {
+    let summary = "Illustrates how to define an operation.";
+    let description = [{
+        The `standalone.foo` operation illustrates how to define a new
+        operation in a dialect. It uses an operation trait to declare that it
+        has no side effects.
+
+        This operation takes an integer argument and returns an integer.
+
+        Example:
+
+        ```mlir
+        %0 = constant 2 : i32
+        // Apply the foo operation to %0
+        %1 = standalone.foo %0 : i32
+        ```
+    }];
+
+    let arguments = (ins I32:$input);
+    let results = (outs I32:$res);
+
+    let assemblyFormat = [{
+        $input attr-dict `:` type($input)
+    }];
+}
+
+#endif // STANDALONE_OPS

mlir/examples/standalone/lib/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(Standalone)

mlir/examples/standalone/lib/Standalone/CMakeLists.txt

@@ -0,0 +1,12 @@
+add_mlir_dialect_library(MLIRStandalone
+        StandaloneDialect.cpp
+        StandaloneOps.cpp
+
+        ADDITIONAL_HEADER_DIRS
+        ${PROJECT_SOURCE_DIR}/include/Standalone
+
+        DEPENDS
+        MLIRStandaloneOpsIncGen
+        )
+
+target_link_libraries(MLIRStandalone PUBLIC MLIRIR)

mlir/examples/standalone/lib/Standalone/StandaloneDialect.cpp

@@ -0,0 +1,25 @@
+//===- StandaloneDialect.cpp - Standalone dialect ---------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Standalone/StandaloneDialect.h"
+#include "Standalone/StandaloneOps.h"
+
+using namespace mlir;
+using namespace mlir::standalone;
+
+//===----------------------------------------------------------------------===//
+// Standalone dialect.
+//===----------------------------------------------------------------------===//
+
+StandaloneDialect::StandaloneDialect(mlir::MLIRContext *context)
+    : Dialect(getDialectNamespace(), context) {
+  addOperations<
+#define GET_OP_LIST
+#include "Standalone/StandaloneOps.cpp.inc"
+      >();
+}

mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp

@@ -0,0 +1,18 @@
+//===- StandaloneOps.cpp - Standalone dialect ops ---------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Standalone/StandaloneOps.h"
+#include "Standalone/StandaloneDialect.h"
+#include "mlir/IR/OpImplementation.h"
+
+namespace mlir {
+namespace standalone {
+#define GET_OP_CLASSES
+#include "Standalone/StandaloneOps.cpp.inc"
+} // namespace standalone
+} // namespace mlir

mlir/examples/standalone/standalone-opt/CMakeLists.txt

@@ -0,0 +1,13 @@
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
+set(LIBS
+        ${dialect_libs}
+        ${conversion_libs}
+        MLIRQuantizerTransforms
+        MLIROptLib
+        MLIRStandalone
+        )
+add_llvm_executable(standalone-opt standalone-opt.cpp)
+
+llvm_update_compile_flags(standalone-opt)
+target_link_libraries(standalone-opt PRIVATE ${LIBS})

mlir/examples/standalone/standalone-opt/standalone-opt.cpp

@@ -0,0 +1,97 @@
+//===- standalone-opt.cpp ---------------------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/InitAllDialects.h"
+#include "mlir/InitAllPasses.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Support/FileUtilities.h"
+#include "mlir/Support/MlirOptMain.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+#include "Standalone/StandaloneDialect.h"
+
+static llvm::cl::opt<std::string> inputFilename(llvm::cl::Positional,
+                                                llvm::cl::desc("<input file>"),
+                                                llvm::cl::init("-"));
+
+static llvm::cl::opt<std::string>
+    outputFilename("o", llvm::cl::desc("Output filename"),
+                   llvm::cl::value_desc("filename"), llvm::cl::init("-"));
+
+static llvm::cl::opt<bool> splitInputFile(
+    "split-input-file",
+    llvm::cl::desc("Split the input file into pieces and process each "
+                   "chunk independently"),
+    llvm::cl::init(false));
+
+static llvm::cl::opt<bool> verifyDiagnostics(
+    "verify-diagnostics",
+    llvm::cl::desc("Check that emitted diagnostics match "
+                   "expected-* lines on the corresponding line"),
+    llvm::cl::init(false));
+
+static llvm::cl::opt<bool> verifyPasses(
+    "verify-each",
+    llvm::cl::desc("Run the verifier after each transformation pass"),
+    llvm::cl::init(true));
+
+static llvm::cl::opt<bool>
+    showDialects("show-dialects",
+                 llvm::cl::desc("Print the list of registered dialects"),
+                 llvm::cl::init(false));
+
+int main(int argc, char **argv) {
+  mlir::registerAllDialects();
+  mlir::registerAllPasses();
+
+  mlir::registerDialect<mlir::standalone::StandaloneDialect>();
+  // TODO: Register standalone passes here.
+
+  llvm::InitLLVM y(argc, argv);
+
+  // Register any pass manager command line options.
+  mlir::registerPassManagerCLOptions();
+  mlir::PassPipelineCLParser passPipeline("", "Compiler passes to run");
+
+  // Parse pass names in main to ensure static initialization completed.
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "MLIR modular optimizer driver\n");
+
+  mlir::MLIRContext context;
+  if (showDialects) {
+    llvm::outs() << "Registered Dialects:\n";
+    for (mlir::Dialect *dialect : context.getRegisteredDialects()) {
+      llvm::outs() << dialect->getNamespace() << "\n";
+    }
+    return 0;
+  }
+
+  // Set up the input file.
+  std::string errorMessage;
+  auto file = mlir::openInputFile(inputFilename, &errorMessage);
+  if (!file) {
+    llvm::errs() << errorMessage << "\n";
+    return 1;
+  }
+
+  auto output = mlir::openOutputFile(outputFilename, &errorMessage);
+  if (!output) {
+    llvm::errs() << errorMessage << "\n";
+    exit(1);
+  }
+
+  return failed(mlir::MlirOptMain(output->os(), std::move(file), passPipeline,
+                                  splitInputFile, verifyDiagnostics,
+                                  verifyPasses));
+}

mlir/examples/standalone/test/CMakeLists.txt

@@ -0,0 +1,19 @@
+configure_lit_site_cfg(
+        ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+        ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+        MAIN_CONFIG
+        ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+
+set(STANDALONE_OPT_TEST_DEPENDS
+        FileCheck count not
+        standalone-opt
+        )
+
+add_lit_testsuite(check-standalone-opt "Running the standalone-opt regression tests"
+        ${CMAKE_CURRENT_BINARY_DIR}
+        DEPENDS ${STANDALONE_OPT_TEST_DEPENDS}
+        )
+set_target_properties(check-standalone-opt PROPERTIES FOLDER "Tests")
+
+add_lit_testsuites(STANDALONE_OPT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${STANDALONE_OPT_TEST_DEPENDS})

mlir/examples/standalone/test/Standalone/dummy.mlir

@@ -0,0 +1,11 @@
+// RUN: standalone-opt %s | standalone-opt | FileCheck %s
+
+module {
+    // CHECK-LABEL: func @bar()
+    func @bar() {
+        %0 = constant 1 : i32
+        // CHECK: %{{.*}} = standalone.foo %{{.*}} : i32
+        %res = standalone.foo %0 : i32
+        return
+    }
+}

mlir/examples/standalone/test/lit.cfg.py

@@ -0,0 +1,60 @@
+# -*- Python -*-
+
+import os
+import platform
+import re
+import subprocess
+import tempfile
+
+import lit.formats
+import lit.util
+
+from lit.llvm import llvm_config
+from lit.llvm.subst import ToolSubst
+from lit.llvm.subst import FindTool
+
+# Configuration file for the 'lit' test runner.
+
+# name: The name of this test suite.
+config.name = 'STANDALONE_OPT'
+
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.mlir']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.standalone_obj_root, 'test')
+
+config.substitutions.append(('%PATH%', config.environment['PATH']))
+config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
+
+llvm_config.with_system_environment(
+    ['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP'])
+
+llvm_config.use_default_substitutions()
+
+# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
+# subdirectories contain auxiliary inputs for various tests in their parent
+# directories.
+config.excludes = ['Inputs', 'Examples', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.standalone_obj_root, 'test')
+config.standalone_tools_dir = os.path.join(config.standalone_obj_root, 'bin')
+
+# Tweak the PATH to include the tools dir.
+llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
+
+tool_dirs = [config.standalone_tools_dir, config.llvm_tools_dir]
+tools = [
+    'standalone-opt'
+]
+
+llvm_config.add_tool_substitutions(tools, tool_dirs)

mlir/examples/standalone/test/lit.site.cfg.py.in

@@ -0,0 +1,49 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.host_triple = "@LLVM_HOST_TRIPLE@"
+config.target_triple = "@TARGET_TRIPLE@"
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
+config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_shlib_ext = "@SHLIBEXT@"
+config.llvm_exe_ext = "@EXEEXT@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.python_executable = "@PYTHON_EXECUTABLE@"
+config.gold_executable = "@GOLD_EXECUTABLE@"
+config.ld64_executable = "@LD64_EXECUTABLE@"
+config.enable_shared = @ENABLE_SHARED@
+config.enable_assertions = @ENABLE_ASSERTIONS@
+config.targets_to_build = "@TARGETS_TO_BUILD@"
+config.native_target = "@LLVM_NATIVE_ARCH@"
+config.llvm_bindings = "@LLVM_BINDINGS@".split(' ')
+config.host_os = "@HOST_OS@"
+config.host_cc = "@HOST_CC@"
+config.host_cxx = "@HOST_CXX@"
+# Note: ldflags can contain double-quoted paths, so must use single quotes here.
+config.host_ldflags = '@HOST_LDFLAGS@'
+config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
+config.llvm_host_triple = '@LLVM_HOST_TRIPLE@'
+config.host_arch = "@HOST_ARCH@"
+config.standalone_src_root = "@CMAKE_SOURCE_DIR@"
+config.standalone_obj_root = "@CMAKE_BINARY_DIR@"
+
+# Support substitution of the tools_dir with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
+except KeyError:
+    e = sys.exc_info()[1]
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@CMAKE_SOURCE_DIR@/test/lit.cfg.py")