[ASan] Add use_madv_dontdump flag.

Differential Revision: http://reviews.llvm.org/D7294

llvm-svn: 227959

compiler-rt/lib/asan/asan_rtl.cc

@@ -100,6 +100,8 @@ void ReserveShadowMemoryRange(uptr beg, uptr end) {
   }
   if (common_flags()->no_huge_pages_for_shadow)
     NoHugePagesInRegion(beg, size);
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(beg, size);
 }
 
 // --------------- LowLevelAllocateCallbac ---------- {{{1

compiler-rt/lib/msan/msan_linux.cc

@@ -119,6 +119,8 @@ bool InitShadow(bool map_shadow, bool init_origins) {
         (init_origins && type == MappingDesc::ORIGIN)) {
       if (!CheckMemoryRangeAvailability(start, size)) return false;
       if ((uptr)MmapFixedNoReserve(start, size) != start) return false;
+      if (common_flags()->use_madv_dontdump)
+        DontDumpShadowMemory(start, size);
     } else if (type == MappingDesc::INVALID) {
       if (!CheckMemoryRangeAvailability(start, size)) return false;
       if (!ProtectMemoryRange(start, size)) return false;

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -77,6 +77,7 @@ void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);
 uptr GetRSS();
 void NoHugePagesInRegion(uptr addr, uptr length);
+void DontDumpShadowMemory(uptr addr, uptr length);
 
 // InternalScopedBuffer can be used instead of large stack arrays to
 // keep frame size low.

compiler-rt/lib/sanitizer_common/sanitizer_flags.inc

@@ -136,6 +136,9 @@ COMMON_FLAG(
     "Disable core dumping. By default, disable_core=1 on 64-bit to avoid "
     "dumping a 16T+ core file. Ignored on OSes that don't dump core by"
     "default and for sanitizers that don't reserve lots of virtual memory.")
+COMMON_FLAG(bool, use_madv_dontdump, true,
+          "If set, instructs kernel to not store the (huge) shadow "
+          "in core file.")
 COMMON_FLAG(bool, symbolize_inline_frames, true,
             "Print inlined frames in stacktraces. Defaults to true.")
 COMMON_FLAG(const char *, stack_trace_format, "DEFAULT",

compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc

@@ -50,6 +50,12 @@ void NoHugePagesInRegion(uptr addr, uptr size) {
 #endif  // MADV_NOHUGEPAGE
 }
 
+void DontDumpShadowMemory(uptr addr, uptr length) {
+#ifdef MADV_DONTDUMP
+  madvise((void *)addr, length, MADV_DONTDUMP);
+#endif
+}
+
 static rlim_t getlim(int res) {
   rlimit rlim;
   CHECK_EQ(0, getrlimit(res, &rlim));

compiler-rt/lib/sanitizer_common/sanitizer_win.cc

@@ -134,13 +134,18 @@ void *Mprotect(uptr fixed_addr, uptr size) {
 
 void FlushUnneededShadowMemory(uptr addr, uptr size) {
   // This is almost useless on 32-bits.
-  // FIXME: add madvice-analog when we move to 64-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
 }
 
 void NoHugePagesInRegion(uptr addr, uptr size) {
   // FIXME: probably similar to FlushUnneededShadowMemory.
 }
 
+void DontDumpShadowMemory(uptr addr, uptr length) {
+  // This is almost useless on 32-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
+}
+
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MEMORY_BASIC_INFORMATION mbi;
   CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc

@@ -218,6 +218,8 @@ void InitializeShadowMemory() {
   // FIXME: don't use constants here.
   NoHugePagesInRegion(MemToShadow(0x7f0000000000ULL),
                       0x10000000000ULL * kShadowMultiplier);
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg);
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       kShadowBeg, kShadowEnd,
       (kShadowEnd - kShadowBeg) >> 30);
@@ -231,6 +233,8 @@ void InitializeShadowMemory() {
                "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg);
     Die();
   }
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);
 

compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc

@@ -60,6 +60,8 @@ void InitializeShadowMemory() {
            "to link with -pie.\n");
     Die();
   }
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg);
   DPrintf("kShadow %zx-%zx (%zuGB)\n",
       kShadowBeg, kShadowEnd,
       (kShadowEnd - kShadowBeg) >> 30);