[TSan] Initialize flags as early as possible. Disables back coredump, accidentally enabled in r215479. Add a test.

llvm-svn: 215763

compiler-rt/lib/tsan/rtl/tsan_platform.h

@@ -128,7 +128,7 @@ void FlushShadowMemory();
 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
 uptr GetRSS();
 
-const char *InitializePlatform();
+void InitializePlatform();
 void FinalizePlatform();
 
 // The additional page is to catch shadow stack overflow as paging fault.

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc

@@ -332,7 +332,7 @@ static void InitDataSeg() {
 
 #endif  // #ifndef TSAN_GO
 
-const char *InitializePlatform() {
+void InitializePlatform() {
   DisableCoreDumperIfNecessary();
 
   // Go maps shadow memory lazily and works fine with limited address space.
@@ -369,7 +369,6 @@ const char *InitializePlatform() {
   InitTlsSize();
   InitDataSeg();
 #endif
-  return GetEnv(kTsanOptionsEnv);
 }
 
 bool IsGlobalVar(uptr addr) {

compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc

@@ -73,9 +73,8 @@ void InitializeShadowMemory() {
 }
 #endif
 
-const char *InitializePlatform() {
+void InitializePlatform() {
   DisableCoreDumperIfNecessary();
-  return GetEnv(kTsanOptionsEnv);
 }
 
 void FinalizePlatform() {

compiler-rt/lib/tsan/rtl/tsan_platform_windows.cc

@@ -35,8 +35,7 @@ uptr GetRSS() {
   return 0;
 }
 
-const char *InitializePlatform() {
-  return GetEnv(kTsanOptionsEnv);
+void InitializePlatform() {
 }
 
 void FinalizePlatform() {

compiler-rt/lib/tsan/rtl/tsan_rtl.cc

@@ -289,18 +289,19 @@ void Initialize(ThreadState *thr) {
   // Install tool-specific callbacks in sanitizer_common.
   SetCheckFailedCallback(TsanCheckFailed);
 
+  ctx = new(ctx_placeholder) Context;
+  const char *options = GetEnv(kTsanOptionsEnv);
+  InitializeFlags(&ctx->flags, options);
 #ifndef TSAN_GO
   InitializeAllocator();
 #endif
   InitializeInterceptors();
-  const char *env = InitializePlatform();
+  InitializePlatform();
   InitializeMutex();
   InitializeDynamicAnnotations();
-  ctx = new(ctx_placeholder) Context;
 #ifndef TSAN_GO
   InitializeShadowMemory();
 #endif
-  InitializeFlags(&ctx->flags, env);
   // Setup correct file descriptor for error reports.
   __sanitizer_set_report_path(flags()->log_path);
   InitializeSuppressions();
@@ -336,7 +337,6 @@ void Initialize(ThreadState *thr) {
 }
 
 int Finalize(ThreadState *thr) {
-  Context *ctx = __tsan::ctx;
   bool failed = false;
 
   if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)

compiler-rt/test/sanitizer_common/TestCases/corelimit.cc

@@ -0,0 +1,16 @@
+// RUN: %clangxx -O0 %s -o %t && %run %t
+// XFAIL: lsan
+
+#include <assert.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+int main() {
+  struct rlimit lim_core;
+  getrlimit(RLIMIT_CORE, &lim_core);
+  void *p;
+  if (sizeof(p) == 8) {
+    assert(0 == lim_core.rlim_max);
+  }
+  return 0;
+}