[sanitizer] Introduce a vDSO aware time function, and use it in the allocator [redo]

Summary:
Redo of D40657, which had the initial discussion. The initial code had to move
into a libcdep file, and things had to be shuffled accordingly.

`NanoTime` is a time sink when checking whether or not to release memory to
the OS. While reducing the amount of calls to said function is in the works,
another solution that was found to be beneficial was to use a timing function
that can leverage the vDSO.

We hit a couple of snags along the way, like the fact that the glibc crashes
when clock_gettime is called from a preinit_array, or the fact that
`__vdso_clock_gettime` is mangled (for security purposes) and can't be used
directly, and also that clock_gettime can be intercepted.

The proposed solution takes care of all this as far as I can tell, and
significantly improve performances and some Scudo load tests with memory
reclaiming enabled.

@mcgrathr: please feel free to follow up on
https://reviews.llvm.org/D40657#940857 here. I posted a reply at
https://reviews.llvm.org/D40657#940974.

Reviewers: alekseyshl, krytarowski, flowerhack, mcgrathr, kubamracek

Reviewed By: alekseyshl, krytarowski

Subscribers: #sanitizers, mcgrathr, srhines, llvm-commits, kubamracek

Differential Revision: https://reviews.llvm.org/D40679

llvm-svn: 320409

compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h

@@ -697,7 +697,7 @@ class SizeClassAllocator64 {
         // Do it only when the feature is turned on, to avoid a potentially
         // extraneous syscall.
         if (ReleaseToOSIntervalMs() >= 0)
-          region->rtoi.last_release_at_ns = NanoTime();
+          region->rtoi.last_release_at_ns = MonotonicNanoTime();
       }
       // Do the mmap for the user memory.
       const uptr user_map_size =
@@ -827,7 +827,7 @@ class SizeClassAllocator64 {
         return;
 
       if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL >
-          NanoTime()) {
+          MonotonicNanoTime()) {
         return;  // Memory was returned recently.
       }
     }
@@ -844,6 +844,6 @@ class SizeClassAllocator64 {
       region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
       region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
     }
-    region->rtoi.last_release_at_ns = NanoTime();
+    region->rtoi.last_release_at_ns = MonotonicNanoTime();
   }
 };

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -295,6 +295,7 @@ uptr GetTlsSize();
 void SleepForSeconds(int seconds);
 void SleepForMillis(int millis);
 u64 NanoTime();
+u64 MonotonicNanoTime();
 int Atexit(void (*function)(void));
 void SortArray(uptr *array, uptr size);
 void SortArray(u32 *array, uptr size);

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc

@@ -2045,6 +2045,13 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) {
   }
   return res;
 }
+namespace __sanitizer {
+extern "C" {
+int real_clock_gettime(u32 clk_id, void *tp) {
+  return REAL(clock_gettime)(clk_id, tp);
+}
+}  // extern "C"
+}  // namespace __sanitizer
 INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp);

compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc

@@ -51,6 +51,8 @@ unsigned int internal_sleep(unsigned int seconds) {
 
 u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); }
 
+u64 MonotonicNanoTime() { return _zx_time_get(ZX_CLOCK_MONOTONIC); }
+
 uptr internal_getpid() {
   zx_info_handle_basic_t info;
   zx_status_t status =

compiler-rt/lib/sanitizer_common/sanitizer_linux.cc

@@ -459,6 +459,10 @@ u64 NanoTime() {
   return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
 }
 
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+  return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp);
+}
+
 // Like getenv, but reads env directly from /proc (on Linux) or parses the
 // 'environ' array (on FreeBSD) and does not use libc. This function should be
 // called first inside __asan_init.

compiler-rt/lib/sanitizer_common/sanitizer_linux.h

@@ -46,6 +46,7 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
 uptr internal_sigaltstack(const void* ss, void* oss);
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
     __sanitizer_sigset_t *oldset);
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
 
 // Linux-only syscalls.
 #if SANITIZER_LINUX

compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc

@@ -707,6 +707,35 @@ void LogMessageOnPrintf(const char *str) {
 
 #endif  // SANITIZER_LINUX
 
+// glibc cannot use clock_gettime from a preinit_array function as the vDSO
+// function pointers haven't been initialized yet. To prevent a crash, we check
+// for the presence of the glibc symbol __vdso_clock_gettime, and verify that it
+// is not null (it can be mangled so we can't use it directly). Bionic's
+// clock_gettime actually falls back to the syscall in the same situation.
+extern "C" SANITIZER_WEAK_ATTRIBUTE void *__vdso_clock_gettime;
+bool CanUseLibcClockGetTime() {
+  return !SANITIZER_FREEBSD && !SANITIZER_NETBSD &&
+      (SANITIZER_ANDROID || (&__vdso_clock_gettime && __vdso_clock_gettime));
+}
+
+// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
+// clock_gettime. real_clock_gettime only exists if clock_gettime is
+// intercepted, so define it weakly and use it if available.
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int real_clock_gettime(u32 clk_id, void *tp);
+u64 MonotonicNanoTime() {
+  timespec ts;
+  if (CanUseLibcClockGetTime()) {
+    if (&real_clock_gettime)
+      real_clock_gettime(CLOCK_MONOTONIC, &ts);
+    else
+      clock_gettime(CLOCK_MONOTONIC, &ts);
+  } else {
+    internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+  }
+  return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+
 } // namespace __sanitizer
 
 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX

compiler-rt/lib/sanitizer_common/sanitizer_mac.cc

@@ -365,6 +365,10 @@ u64 NanoTime() {
   return 0;
 }
 
+u64 MonotonicNanoTime() {
+  return 0;
+}
+
 uptr GetTlsSize() {
   return 0;
 }

compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc

@@ -36,6 +36,7 @@
 # define SYS_sigaltstack SYS___sigaltstack14
 # define SYS_sigprocmask SYS___sigprocmask14
 # define SYS_nanosleep SYS___nanosleep50
+# define SYS_clock_gettime SYS___clock_gettime50
 # if SANITIZER_WORDSIZE == 64
 #  define internal_syscall_ptr  __syscall
 # else

compiler-rt/lib/sanitizer_common/sanitizer_win.cc

@@ -505,6 +505,10 @@ u64 NanoTime() {
   return 0;
 }
 
+u64 MonotonicNanoTime() {
+  return 0;
+}
+
 void Abort() {
   internal__exit(3);
 }

compiler-rt/lib/scudo/scudo_allocator.cpp

@@ -301,7 +301,7 @@ struct ScudoAllocator {
 
     CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
     if (CheckRssLimit)
-      atomic_store_relaxed(&RssLastCheckedAtNS, NanoTime());
+      atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime());
   }
 
   // Helper function that checks for a valid Scudo chunk. nullptr isn't.
@@ -319,7 +319,7 @@ struct ScudoAllocator {
   // it can, every 100ms, otherwise it will just return the current one.
   bool isRssLimitExceeded() {
     u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
-    const u64 CurrentCheck = NanoTime();
+    const u64 CurrentCheck = MonotonicNanoTime();
     if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
       return atomic_load_relaxed(&RssLimitExceeded);
     if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,

compiler-rt/lib/scudo/scudo_tsd.h

@@ -36,7 +36,7 @@ struct ALIGNED(64) ScudoTSD {
       return true;
     }
     if (atomic_load_relaxed(&Precedence) == 0)
-      atomic_store_relaxed(&Precedence, NanoTime());
+      atomic_store_relaxed(&Precedence, MonotonicNanoTime());
     return false;
   }