Improve TestConcurrentEvents.py

- code cleanup, improved reporting when failures take place
- ensure known thread is interrupted by using pthread_kill() instead of kill() in the signal worker thread
- above should avoid llvm.org/pr16567 on Mac OS X (though kill() could still cause threads to pop out of existance temporarily)
- added an additional check that all threads have exited after worker threads are all join()ed
- logged llvm.org/pr16603 for the new Linux bug discovered

llvm-svn: 186124

lldb/test/functionalities/thread/concurrent_events/TestConcurrentEvents.py

@@ -10,12 +10,19 @@ until exit or a crash takes place, and the number of events seen by LLDB is
 verified to match the expected number of events.
 """
 
-import os, time
+import os, signal, time
 import unittest2
 import lldb
 from lldbtest import *
 import lldbutil
 
+# ==================================================
+# Dictionary of signal names
+# ==================================================
+signal_names = dict((getattr(signal, n), n) \
+        for n in dir(signal) if n.startswith('SIG') and '_' not in n )
+
+
 class ConcurrentEventsTestCase(TestBase):
 
     mydir = os.path.join("functionalities", "thread", "concurrent_events")
@@ -321,27 +328,54 @@ class ConcurrentEventsTestCase(TestBase):
                                num_breakpoint_threads=1,
                                num_signal_threads=1)
 
-
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
         # Find the line number for our breakpoint.
-        self.thread_breakpoint = line_number('main.cpp', '// Set breakpoint here')
+        self.filename = 'main.cpp'
-        self.setup_breakpoint = line_number('main.cpp', '// Break here and adjust num')
+        self.thread_breakpoint_line = line_number(self.filename, '// Set breakpoint here')
+        self.setup_breakpoint_line = line_number(self.filename, '// Break here and adjust num')
+        self.finish_breakpoint_line = line_number(self.filename, '// Break here and verify one thread is active')
 
-    def print_threads(self, threads):
+    def describe_threads(self):
-        ret = ""
+        ret = []
-        for x in threads:
+        for x in self.inferior_process:
-            ret += "\t thread %d stopped due to reason %s" % (x.GetIndexID(), lldbutil.stop_reason_to_str(x.GetStopReason()))
+            id = x.GetIndexID()
+            reason = x.GetStopReason()
+            status = "stopped" if x.IsStopped() else "running"
+            reason_str = lldbutil.stop_reason_to_str(reason)
+            if reason == lldb.eStopReasonBreakpoint:
+                bpid = x.GetStopReasonDataAtIndex(0)
+                bp = self.inferior_target.FindBreakpointByID(bpid)
+                reason_str = "%s hit %d times" % (lldbutil.get_description(bp), bp.GetHitCount())
+            elif reason == lldb.eStopReasonWatchpoint:
+                watchid = x.GetStopReasonDataAtIndex(0)
+                watch = self.inferior_target.FindWatchpointByID(watchid)
+                reason_str = "%s hit %d times" % (lldbutil.get_description(watch), watch.GetHitCount())
+            elif reason == lldb.eStopReasonSignal:
+                reason_str = "signal %s" % (signal_names[x.GetStopReasonDataAtIndex(0)])
+
+            location = "\t".join([lldbutil.get_description(x.GetFrameAtIndex(i)) for i in range(x.GetNumFrames())])
+            ret.append("thread %d %s due to %s at\n\t%s" % (id, status, reason_str, location))
         return ret
 
-    def debug_threads(self, bps, crashed, exiting, wps, signals, others):
+    def add_breakpoint(self, line, descriptions):
-        print "%d threads stopped at bp:\n%s" % (len(bps), self.print_threads(bps))
+        """ Adds a breakpoint at self.filename:line and appends its description to descriptions, and
-        print "%d threads crashed:\n%s" % (len(crashed), self.print_threads(crashed))
+            returns the LLDB SBBreakpoint object.
-        print "%d threads stopped due to watchpoint:\n%s" % (len(wps), self.print_threads(wps))
+        """
-        print "%d threads stopped at signal:\n%s" % (len(signals), self.print_threads(signals))
+
-        print "%d threads exiting:\n%s" % (len(exiting), self.print_threads(exiting))
+        bpno = lldbutil.run_break_set_by_file_and_line(self, self.filename, line, num_expected_locations=1)
-        print "%d threads stopped due to other/unknown reason:\n%s" % (len(others), self.print_threads(others))
+        bp = self.inferior_target.FindBreakpointByID(bpno)
+        descriptions.append(": file = 'main.cpp', line = %d, locations = 1" % self.finish_breakpoint_line)
+        return bp
+
+    def inferior_done(self):
+        """ Returns true if the inferior is done executing all the event threads (and is stopped at self.finish_breakpoint, 
+            or has terminated execution.
+        """
+        return self.finish_breakpoint.GetHitCount() > 0 or \
+                self.crash_count > 0 or \
+                self.inferior_process.GetState == lldb.eStateExited
 
     def do_thread_actions(self,
                           num_breakpoint_threads = 0,
@@ -361,16 +395,21 @@ class ConcurrentEventsTestCase(TestBase):
         exe = os.path.join(os.getcwd(), "a.out")
         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
 
-        # Initialize all the breakpoints (main thread/aux thread)
+        # Get the target
-        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.setup_breakpoint,
+        self.inferior_target = self.dbg.GetSelectedTarget()
-            num_expected_locations=1)
-        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.thread_breakpoint,
-            num_expected_locations=1)
 
-        # The breakpoint list should show 2 breakpoints with 1 location.
+        expected_bps = []
-        self.expect("breakpoint list -f", "Breakpoint location shown correctly",
+
-            substrs = ["1: file = 'main.cpp', line = %d, locations = 1" % self.setup_breakpoint,
+        # Initialize all the breakpoints (main thread/aux thread)
-                       "2: file = 'main.cpp', line = %d, locations = 1" % self.thread_breakpoint])
+        self.setup_breakpoint = self.add_breakpoint(self.setup_breakpoint_line, expected_bps)
+        self.finish_breakpoint = self.add_breakpoint(self.finish_breakpoint_line, expected_bps)
+
+        # Set the thread breakpoint
+        if num_breakpoint_threads + num_delay_breakpoint_threads > 0:
+            self.thread_breakpoint = self.add_breakpoint(self.thread_breakpoint_line, expected_bps)
+
+        # Verify breakpoints
+        self.expect("breakpoint list -f", "Breakpoint locations shown correctly", substrs = expected_bps)
 
         # Run the program.
         self.runCmd("run", RUN_SUCCEEDED)
@@ -379,17 +418,19 @@ class ConcurrentEventsTestCase(TestBase):
         self.expect("thread backtrace", STOPPED_DUE_TO_BREAKPOINT,
             substrs = ["stop reason = breakpoint 1."])
 
-        # Initialize the watchpoint on the global variable (g_watchme)
+        # Initialize the (single) watchpoint on the global variable (g_watchme)
         if num_watchpoint_threads + num_delay_watchpoint_threads > 0:
             self.runCmd("watchpoint set variable g_watchme")
+            for w in self.inferior_target.watchpoint_iter():
+                self.thread_watchpoint = w
+                self.assertTrue("g_watchme" in str(self.thread_watchpoint), "Watchpoint location not shown correctly")
 
-        # Get the target process
+        # Get the process
-        target = self.dbg.GetSelectedTarget()
+        self.inferior_process = self.inferior_target.GetProcess()
-        process = target.GetProcess()
 
         # We should be stopped at the setup site where we can set the number of
         # threads doing each action (break/crash/signal/watch)
-        self.assertEqual(process.GetNumThreads(), 1, 'Expected to stop before any additional threads are spawned.')
+        self.assertEqual(self.inferior_process.GetNumThreads(), 1, 'Expected to stop before any additional threads are spawned.')
 
         self.runCmd("expr num_breakpoint_threads=%d" % num_breakpoint_threads)
         self.runCmd("expr num_crash_threads=%d" % num_crash_threads)
@@ -401,73 +442,68 @@ class ConcurrentEventsTestCase(TestBase):
         self.runCmd("expr num_delay_signal_threads=%d" % num_delay_signal_threads)
         self.runCmd("expr num_delay_watchpoint_threads=%d" % num_delay_watchpoint_threads)
 
+        # Continue the inferior so threads are spawned
         self.runCmd("continue")
 
         # Make sure we see all the threads. The inferior program's threads all synchronize with a pseudo-barrier; that is,
         # the inferior program ensures all threads are started and running before any thread triggers its 'event'.
-        num_threads = process.GetNumThreads()
+        num_threads = self.inferior_process.GetNumThreads()
         expected_num_threads = num_breakpoint_threads + num_delay_breakpoint_threads \
                              + num_signal_threads + num_delay_signal_threads \
                              + num_watchpoint_threads + num_delay_watchpoint_threads \
                              + num_crash_threads + num_delay_crash_threads + 1
         self.assertEqual(num_threads, expected_num_threads,
-            'Number of expected threads and actual threads do not match.')
+            'Expected to see %d threads, but seeing %d. Details:\n%s' % (expected_num_threads,
+                                                                         num_threads,
+                                                                         "\n\t".join(self.describe_threads())))
 
-        # Get the thread objects
+        self.signal_count = len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonSignal))
-        (breakpoint_threads, crashed_threads, exiting_threads, other_threads, signal_threads, watchpoint_threads) = ([], [], [], [], [], [])
+        self.crash_count = len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonException))
-        lldbutil.sort_stopped_threads(process,
-                                      breakpoint_threads=breakpoint_threads,
-                                      crashed_threads=crashed_threads,
-                                      exiting_threads=exiting_threads,
-                                      signal_threads=signal_threads,
-                                      watchpoint_threads=watchpoint_threads,
-                                      other_threads=other_threads)
 
-        if self.TraceOn():
+        # Run to completion (or crash)
-            self.debug_threads(breakpoint_threads, crashed_threads, exiting_threads, watchpoint_threads, signal_threads, other_threads)
+        while not self.inferior_done(): 
-
-        # The threads that are doing signal handling must be unblocked or the inferior will hang. We keep
-        # a counter of threads that stop due to a signal so we have something to verify later on.
-        seen_signal_threads = len(signal_threads)
-        seen_breakpoint_threads = len(breakpoint_threads)
-        seen_watchpoint_threads = len(watchpoint_threads)
-        seen_crashed_threads = len(crashed_threads)
-
-        # Run to completion
-        while len(crashed_threads) == 0 and process.GetState() != lldb.eStateExited:
             if self.TraceOn():
                 self.runCmd("thread backtrace all")
-                self.debug_threads(breakpoint_threads, crashed_threads, exiting_threads, watchpoint_threads, signal_threads, other_threads)
-
             self.runCmd("continue")
-            lldbutil.sort_stopped_threads(process,
+            self.signal_count += len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonSignal))
-                                          breakpoint_threads=breakpoint_threads,
+            self.crash_count += len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonException))
-                                          crashed_threads=crashed_threads,
-                                          exiting_threads=exiting_threads,
-                                          signal_threads=signal_threads,
-                                          watchpoint_threads=watchpoint_threads,
-                                          other_threads=other_threads)
-            seen_signal_threads += len(signal_threads)
-            seen_breakpoint_threads += len(breakpoint_threads)
-            seen_watchpoint_threads += len(watchpoint_threads)
-            seen_crashed_threads += len(crashed_threads)
 
         if num_crash_threads > 0 or num_delay_crash_threads > 0:
             # Expecting a crash
-            self.assertTrue(seen_crashed_threads > 0, "Expecting at least one thread to crash")
+            self.assertTrue(self.crash_count > 0,
+                "Expecting at least one thread to crash. Details: %s" % "\t\n".join(self.describe_threads()))
 
             # Ensure the zombie process is reaped
             self.runCmd("process kill")
 
         elif num_crash_threads == 0 and num_delay_crash_threads == 0:
+            # There should be a single active thread (the main one) which hit the breakpoint after joining
+            self.assertEqual(1, self.finish_breakpoint.GetHitCount(), "Expected main thread (finish) breakpoint to be hit once")
+
+            # llvm.org/pr16603 -- LLDB on Linux sometimes reports exited threads as still 'running'
+            #num_threads = self.inferior_process.GetNumThreads()
+            #self.assertEqual(1, num_threads, "Expecting 1 thread but seeing %d. Details:%s" % (num_threads,
+            #                                                                                 "\n\t".join(self.describe_threads())))
+            self.runCmd("continue")
+
             # The inferior process should have exited without crashing
-            self.assertEqual(0, seen_crashed_threads, "Unexpected thread(s) in crashed state")
+            self.assertEqual(0, self.crash_count, "Unexpected thread(s) in crashed state")
-            self.assertTrue(process.GetState() == lldb.eStateExited, PROCESS_EXITED)
+            self.assertTrue(self.inferior_process.GetState() == lldb.eStateExited, PROCESS_EXITED)
 
             # Verify the number of actions took place matches expected numbers
-            self.assertEqual(num_delay_breakpoint_threads + num_breakpoint_threads, seen_breakpoint_threads)
+            expected_breakpoint_threads = num_delay_breakpoint_threads + num_breakpoint_threads
-            self.assertEqual(num_delay_signal_threads + num_signal_threads, seen_signal_threads)
+            breakpoint_hit_count = self.thread_breakpoint.GetHitCount() if expected_breakpoint_threads > 0 else 0
-            self.assertEqual(num_delay_watchpoint_threads + num_watchpoint_threads, seen_watchpoint_threads)
+            self.assertEqual(expected_breakpoint_threads, breakpoint_hit_count,
+                "Expected %d breakpoint hits, but got %d" % (expected_breakpoint_threads, breakpoint_hit_count))
+
+            expected_signal_threads = num_delay_signal_threads + num_signal_threads
+            self.assertEqual(expected_signal_threads, self.signal_count,
+                "Expected %d stops due to signal delivery, but got %d" % (expected_breakpoint_threads, self.signal_count))
+
+            expected_watchpoint_threads = num_delay_watchpoint_threads + num_watchpoint_threads
+            watchpoint_hit_count = self.thread_watchpoint.GetHitCount() if expected_watchpoint_threads > 0 else 0
+            self.assertEqual(expected_watchpoint_threads, watchpoint_hit_count,
+                "Expected %d watchpoint hits, got %d" % (expected_watchpoint_threads, watchpoint_hit_count))
 
 
 if __name__ == '__main__':

lldb/test/functionalities/thread/concurrent_events/main.cpp

@@ -7,10 +7,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This test is intended to create a situation in which a watchpoint will be hit
+// This test is intended to create a situation in which multiple events
-// while a breakpoint is being handled in another thread.  The expected result is
+// (breakpoints, watchpoints, crashes, and signal generation/delivery) happen
-// that the watchpoint in the second thread will be hit while the breakpoint handler
+// from multiple threads. The test expects the debugger to set a breakpoint on
-// in the first thread is trying to stop all threads.
+// the main thread (before any worker threads are spawned) and modify variables
+// which control the number of therads that are spawned for each action.
 
 #include <atomic>
 #include <vector>
@@ -72,8 +73,10 @@ signal_func (void *input) {
     pseudo_barrier_wait(g_barrier);
     do_action_args(input);
 
-    // Generate a user-defined signal to current process
+    // Send a user-defined signal to the current process
-    kill(getpid(), SIGUSR1);
+    //kill(getpid(), SIGUSR1);
+    // Send a user-defined signal to the current thread
+    pthread_kill(pthread_self(), SIGUSR1);
 
     return 0;
 }
@@ -133,17 +136,20 @@ int main ()
 {
     g_watchme = 0;
 
+    // Actions are triggered immediately after the thread is spawned
     unsigned num_breakpoint_threads = 1;
-    unsigned num_watchpoint_threads = 0;
+    unsigned num_watchpoint_threads = 1;
     unsigned num_signal_threads = 0;
-    unsigned num_crash_threads = 1;
+    unsigned num_crash_threads = 0;
 
+    // Actions below are triggered after a 1-second delay
     unsigned num_delay_breakpoint_threads = 0;
-    unsigned num_delay_watchpoint_threads = 0;
+    unsigned num_delay_watchpoint_threads = 1;
     unsigned num_delay_signal_threads = 0;
     unsigned num_delay_crash_threads = 0;
 
-    // Break here and adjust num_[breakpoint|watchpoint|signal|crash]_threads
+    thread_vector threads; // Break here and adjust num_[breakpoint|watchpoint|signal|crash]_threads
+
     unsigned total_threads = num_breakpoint_threads \
                              + num_watchpoint_threads \
                              + num_signal_threads \
@@ -156,7 +162,6 @@ int main ()
     // Don't let either thread do anything until they're both ready.
     pseudo_barrier_init(g_barrier, total_threads);
 
-    thread_vector threads;
 
     action_counts actions;
     actions.push_back(std::make_pair(num_breakpoint_threads, breakpoint_func));
@@ -185,6 +190,5 @@ int main ()
     for(thread_iterator t = threads.begin(); t != threads.end(); ++t)
         pthread_join(*t, 0);
 
-    // Break here and verify one thread is active.
+    return 0; // Break here and verify one thread is active.
-    return 0;
 }