[sanitizer] Change Mmap*NoAccess to return nullptr on error

Summary:
`MmapNoAccess` & `MmapFixedNoAccess` return directly the result of
`internal_mmap`, as opposed to other Mmap functions that return nullptr.

This inconsistency leads to some confusion for the callers, as some check for
`~(uptr)0` (`MAP_FAILED`) for failure (while it can fail with `-ENOMEM` for
example).

Two potential solutions: change the callers, or make the functions return
`nullptr` on failure to follow the precedent set by the other functions.
The second option looked more appropriate to me.

Correct the callers that were wrongly checking for `~(uptr)0` or
`MAP_FAILED`.

TODO for follow up CLs:
- There are a couple of `internal_mmap` calls in XRay that check for
  MMAP_FAILED as a result as well (cc: @dberris); they should use
  `internal_iserror`;


Reviewers: eugenis, alekseyshl, dberris, kubamracek

Reviewed By: alekseyshl

Subscribers: kristina, kubamracek, delcypher, #sanitizers, dberris, llvm-commits

Differential Revision: https://reviews.llvm.org/D50940

llvm-svn: 340576

compiler-rt/lib/asan/asan_linux.cc

@@ -126,7 +126,7 @@ uptr FindDynamicShadowStart() {
   uptr map_size = shadow_size + left_padding + alignment;
 
   uptr map_start = (uptr)MmapNoAccess(map_size);
-  CHECK_NE(map_start, ~(uptr)0);
+  CHECK(map_start);
 
   uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
   UnmapFromTo(map_start, shadow_start - left_padding);

compiler-rt/lib/asan/asan_premap_shadow.cc

@@ -40,7 +40,7 @@ uptr PremapShadow() {
   uptr map_size = shadow_size + left_padding + alignment;
 
   uptr map_start = (uptr)MmapNoAccess(map_size);
-  CHECK_NE(map_start, ~(uptr)0);
+  CHECK(map_start);
 
   uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
   uptr shadow_end = shadow_start + shadow_size;

compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc

@@ -46,7 +46,7 @@ static uptr MapDynamicShadow(uptr shadow_size_bytes) {
   const uptr map_size = shadow_size + left_padding + alignment;
 
   const uptr map_start = (uptr)MmapNoAccess(map_size);
-  CHECK_NE(map_start, ~(uptr)0);
+  CHECK(map_start);
 
   const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
 

compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h

@@ -77,7 +77,7 @@ class SizeClassAllocator64 {
     } else {
       NonConstSpaceBeg = address_range.Init(TotalSpaceSize,
                                             PrimaryAllocatorName);
-      CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+      CHECK(NonConstSpaceBeg);
     }
     SetReleaseToOSIntervalMs(release_to_os_interval_ms);
     MapWithCallbackOrDie(SpaceEnd(), AdditionalSize());

compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc

@@ -15,14 +15,12 @@
 #if SANITIZER_MAC
 #include "sanitizer_mac.h"
 
-#include <sys/mman.h>
-
 namespace __sanitizer {
 
 void RestrictMemoryToMaxAddress(uptr max_address) {
   uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address;
   void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
-  CHECK(res != MAP_FAILED);
+  CHECK(res);
 }
 
 }  // namespace __sanitizer

compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc

@@ -387,13 +387,18 @@ void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
   unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
   if (fd == -1) flags |= MAP_ANON;
 
-  return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd,
-                               0);
+  uptr p = internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd, 0);
+  if (internal_iserror(p))
+    return nullptr;
+  return (void *)p;
 }
 
 void *MmapNoAccess(uptr size) {
   unsigned flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
-  return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
+  uptr p = internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
+  if (internal_iserror(p))
+    return nullptr;
+  return (void *)p;
 }
 
 // This function is defined elsewhere if we intercepted pthread_attr_getstack.

compiler-rt/lib/scudo/scudo_allocator_secondary.h

@@ -86,7 +86,7 @@ class LargeMmapAllocator {
 
     ReservedAddressRange AddressRange;
     uptr ReservedBeg = AddressRange.Init(ReservedSize, SecondaryAllocatorName);
-    if (UNLIKELY(ReservedBeg == ~static_cast<uptr>(0)))
+    if (UNLIKELY(!ReservedBeg))
       return nullptr;
     // A page-aligned pointer is assumed after that, so check it now.
     DCHECK(IsAligned(ReservedBeg, PageSize));