[LIT] Have lit run the lldb test suite

This is the first in what will hopefully become a series of patches to
replace the driver logic in dotest.py with LIT. The motivation for this
change is that there's no point in maintaining two driver
implementations. Since all of the LLVM projects are using lit, this is
the obvious choice.

Obviously the goal is maintain full compatibility with the functionality
offered by dotest. As such we won't be removing anything until that
point has been reached.

This patch is the initial attempt (referred to as v1) to run the lldb
test suite with lit. To do so we introduced a custom LLDB test format
that invokes dotest.py with a single test file.

Differential revision: https://reviews.llvm.org/D45333

llvm-svn: 330275

lldb/lit/Suite/lit.cfg

@@ -0,0 +1,29 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+
+# name: The name of this test suite.
+config.name = 'lldb-Suite'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.py']
+
+# test_source_root: The root path where tests are located.
+# test_exec_root: The root path where tests should be run.
+config.test_source_root = os.path.join(config.lldb_src_root, 'packages','Python','lldbsuite','test')
+config.test_exec_root = config.test_source_root
+
+# Build dotest command.
+dotest_cmd = [config.dotest_path, '-q']
+dotest_cmd.extend(config.dotest_args_str.split(';'))
+
+# Load LLDB test format.
+sys.path.append(os.path.join(config.lldb_src_root, "lit", "Suite"))
+import lldbtest
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lldbtest.LLDBTest(dotest_cmd)

lldb/lit/Suite/lit.site.cfg.in

@@ -0,0 +1,28 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+config.test_exec_root = "@LLVM_BINARY_DIR@"
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_build_mode = "@LLVM_BUILD_MODE@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.lldb_obj_root = "@LLDB_BINARY_DIR@"
+config.lldb_src_root = "@LLDB_SOURCE_DIR@"
+config.target_triple = "@TARGET_TRIPLE@"
+config.python_executable = "@PYTHON_EXECUTABLE@"
+config.dotest_path = "@LLDB_SOURCE_DIR@/test/dotest.py"
+config.dotest_args_str = "@LLDB_DOTEST_ARGS@"
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.llvm_build_mode = config.llvm_build_mode % lit_config.params
+except KeyError as e:
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@LLDB_SOURCE_DIR@/lit/Suite/lit.cfg")

lldb/lit/Suite/lldbtest.py

@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+import os
+
+import subprocess
+import sys
+
+import lit.Test
+import lit.TestRunner
+import lit.util
+from lit.formats.base import TestFormat
+
+
+class LLDBTest(TestFormat):
+    def __init__(self, dotest_cmd):
+        self.dotest_cmd = dotest_cmd
+
+    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig,
+                            localConfig):
+        source_path = testSuite.getSourcePath(path_in_suite)
+        for filename in os.listdir(source_path):
+            # Ignore dot files and excluded tests.
+            if (filename.startswith('.') or filename in localConfig.excludes):
+                continue
+
+            # Ignore files that don't start with 'Test'.
+            if not filename.startswith('Test'):
+                continue
+
+            filepath = os.path.join(source_path, filename)
+            if not os.path.isdir(filepath):
+                base, ext = os.path.splitext(filename)
+                if ext in localConfig.suffixes:
+                    yield lit.Test.Test(testSuite, path_in_suite +
+                                        (filename, ), localConfig)
+
+    def execute(self, test, litConfig):
+        if litConfig.noExecute:
+            return lit.Test.PASS, ''
+
+        if test.config.unsupported:
+            return (lit.Test.UNSUPPORTED, 'Test is unsupported')
+
+        testPath, testFile = os.path.split(test.getSourcePath())
+        cmd = self.dotest_cmd + [testPath, '-p', testFile]
+
+        try:
+            out, err, exitCode = lit.util.executeCommand(
+                cmd,
+                env=test.config.environment,
+                timeout=litConfig.maxIndividualTestTime)
+        except lit.util.ExecuteCommandTimeoutException:
+            return (lit.Test.TIMEOUT, 'Reached timeout of {} seconds'.format(
+                litConfig.maxIndividualTestTime))
+
+        if exitCode:
+            return lit.Test.FAIL, out + err
+
+        passing_test_line = 'RESULT: PASSED'
+        if passing_test_line not in out and passing_test_line not in err:
+            msg = ('Unable to find %r in dotest output:\n\n%s%s' %
+                   (passing_test_line, out, err))
+            return lit.Test.UNRESOLVED, msg
+
+        return lit.Test.PASS, ''

lldb/test/CMakeLists.txt

@@ -130,11 +130,7 @@ add_python_test_target(check-lldb-single
 
 # If tests crash cause LLDB to crash, or things are otherwise unstable, or if machine-parsable
 # output is desired (i.e. in continuous integration contexts) check-lldb-single is a better target.
-add_python_test_target(check-lldb
-  ${LLDB_SOURCE_DIR}/test/dotest.py
-  "-q;${LLDB_DOTEST_ARGS}"
-  "Testing LLDB (parallel execution, with a separate subprocess per test)"
-  )
+add_custom_target(check-lldb)
 
 # Generate a wrapper for dotest.py in the bin directory.
 # We need configure_file to substitute variables.
@@ -153,6 +149,17 @@ file(GENERATE
 add_custom_target(lldb-dotest)
 add_dependencies(lldb-dotest ${LLDB_TEST_DEPS})
 
+configure_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/../lit/Suite/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/../lit/Suite/lit.site.cfg
+  )
+file(GENERATE
+  OUTPUT
+  ${CMAKE_CURRENT_BINARY_DIR}/../lit/Suite/lit.site.cfg
+  INPUT
+  ${CMAKE_CURRENT_BINARY_DIR}/../lit/Suite/lit.site.cfg
+  )
+
 # If we're building with an in-tree clang, then list clang as a dependency
 # to run tests.
 if (TARGET clang)