[Sanitizers] LSan allocator set errno on failure.

Set proper errno code on alloction failures and change valloc and
memalign implementations to satisfy their man-specified requirements.

llvm-svn: 308063

compiler-rt/lib/lsan/lsan_allocator.cc

@@ -16,6 +16,7 @@
 
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
@@ -86,6 +87,13 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
   return p;
 }
 
+static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) {
+  if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+    return Allocator::FailureHandler::OnBadRequest();
+  size *= nmemb;
+  return Allocate(stack, size, 1, true);
+}
+
 void Deallocate(void *p) {
   if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
   RunFreeHooks(p);
@@ -117,12 +125,22 @@ uptr GetMallocUsableSize(const void *p) {
   return m->requested_size;
 }
 
+inline void *check_ptr(void *ptr) {
+  if (UNLIKELY(!ptr))
+    errno = errno_ENOMEM;
+  return ptr;
+}
+
 void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) {
-  return Allocate(stack, size, alignment, kAlwaysClearMemory);
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return check_ptr(Allocate(stack, size, alignment, kAlwaysClearMemory));
 }
 
 void *lsan_malloc(uptr size, const StackTrace &stack) {
-  return Allocate(stack, size, 1, kAlwaysClearMemory);
+  return check_ptr(Allocate(stack, size, 1, kAlwaysClearMemory));
 }
 
 void lsan_free(void *p) {
@@ -130,20 +148,16 @@ void lsan_free(void *p) {
 }
 
 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
-  return Reallocate(stack, p, size, 1);
+  return check_ptr(Reallocate(stack, p, size, 1));
 }
 
 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
-  if (CheckForCallocOverflow(size, nmemb))
-    return Allocator::FailureHandler::OnBadRequest();
-  size *= nmemb;
-  return Allocate(stack, size, 1, true);
+  return check_ptr(Calloc(nmemb, size, stack));
 }
 
 void *lsan_valloc(uptr size, const StackTrace &stack) {
-  if (size == 0)
-    size = GetPageSizeCached();
-  return Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory);
+  return check_ptr(
+      Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory));
 }
 
 uptr lsan_mz_size(const void *p) {

compiler-rt/test/lsan/TestCases/allocator_returns_null.cc

@@ -37,9 +37,10 @@
 // RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
 
 #include <assert.h>
-#include <string.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <limits>
 #include <new>
 
@@ -86,6 +87,8 @@ int main(int argc, char **argv) {
     assert(0);
   }
 
+  fprintf(stderr, "errno: %d\n", errno);
+
   // The NULL pointer is printed differently on different systems, while (long)0
   // is always the same.
   fprintf(stderr, "x: %zu\n", (size_t)x);
@@ -110,14 +113,19 @@ int main(int argc, char **argv) {
 // CHECK-nnCRASH: Sanitizer's allocator is terminating the process
 
 // CHECK-mNULL: malloc:
+// CHECK-mNULL: errno: 12
 // CHECK-mNULL: x: 0
 // CHECK-cNULL: calloc:
+// CHECK-cNULL: errno: 12
 // CHECK-cNULL: x: 0
 // CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: errno: 12
 // CHECK-coNULL: x: 0
 // CHECK-rNULL: realloc:
+// CHECK-rNULL: errno: 12
 // CHECK-rNULL: x: 0
 // CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: errno: 12
 // CHECK-mrNULL: x: 0
 // CHECK-nnNULL: new-nothrow:
 // CHECK-nnNULL: x: 0