Add a workaround for OS X ignoring RLIMIT_RSS / RLIMIT_AS and getrusage()

not filling in RSS information. After each memory allocation, we ask the
Mach for the RSS value.

configure.ac

@@ -45,7 +45,7 @@ esac
 AC_SUBST(WINSOCK2_LIBS)
 AC_SUBST(DLL_LDFLAGS)
 
-AC_CHECK_HEADERS(windows.h winsock2.h io.h process.h unistd.h inttypes.h stdint.h getopt.h libc.h malloc.h dlfcn.h regex.h sys/cdefs.h sys/socket.h netinet/in.h arpa/inet.h sys/uio.h aio.h sys/mman.h sys/wait.h sys/resource.h sys/time.h endian.h)
+AC_CHECK_HEADERS(windows.h winsock2.h io.h process.h unistd.h inttypes.h stdint.h getopt.h libc.h malloc.h dlfcn.h regex.h sys/cdefs.h sys/socket.h netinet/in.h arpa/inet.h sys/uio.h aio.h sys/mman.h sys/wait.h sys/resource.h sys/time.h endian.h mach/task.h)
 
 AC_CHECK_FUNCS(setenv waitpid setrlimit gettimeofday fork kill pipe _pipe)
 AC_CHECK_FUNCS(open64 __open64 lseek64 __lseek64 mmap64 fopen64 __fopen64 freopen64 __freopen64 dup dup2 ftello ftello64 __ftello64 fseeko fseeko64 __fseeko64 fsetpos64 __fsetpos64 _IO_getc getline getdelim __getdelim fgetln __srefill __filbuf __srget __uflow map_fd memalign posix_memalign aio_read accept bind connect socket readv pread recv recvfrom recvmsg mmap valloc sigaction getpagesize getc_unlocked getchar_unlocked fgetc_unlocked fread_unlocked fgets_unlocked)

msvc/config.h

@@ -59,6 +59,7 @@
 /* #undef HAVE_KILL */
 /* #undef HAVE_LIBC_H */
 /* #undef HAVE_LSEEK64 */
+/* #undef HAVE_MACH_TASK_H */
 #define HAVE_MALLOC_H 1
 /* #undef HAVE_MAP_FD */
 /* #undef HAVE_MEMALIGN */

src/libzzuf/lib-mem.c

@@ -63,6 +63,10 @@
 #if defined HAVE_LIBC_H
 #   include <libc.h>
 #endif
+#if defined HAVE_MACH_TASK_H
+#   include <mach/mach.h>
+#   include <mach/task.h>
+#endif
 
 #include "libzzuf.h"
 #include "lib-load.h"
@@ -125,6 +129,22 @@ static int64_t dummy_offset = 0;
 #define DUMMY_START ((uintptr_t)dummy_buffer)
 #define DUMMY_STOP ((uintptr_t)dummy_buffer + DUMMY_BYTES)
 
+/* setrlimit(RLIMIT_AS) is ignored on OS X, we need to check memory usage
+ * from inside the process. Oh, and getrusage() doesn't work either. */
+static int memory_exceeded(void)
+{
+#if defined HAVE_MACH_TASK_H
+    struct task_basic_info tbi;
+    mach_msg_type_number_t mmtn = TASK_BASIC_INFO_COUNT;
+
+    if (task_info(mach_task_self(), TASK_BASIC_INFO,
+                  (task_info_t)&tbi, &mmtn) == KERN_SUCCESS
+         && tbi.resident_size > _zz_memory)
+        return 1;
+#endif
+    return 0;
+}
+
 void _zz_mem_init(void)
 {
     LOADSYM(free);
@@ -171,7 +191,8 @@ void *NEW(malloc)(size_t size)
         return ret;
     }
     ret = ORIG(malloc)(size);
-    if(ret == NULL && _zz_memory && errno == ENOMEM)
+    if (_zz_memory && ((!ret && errno == ENOMEM)
+                        || (ret && memory_exceeded())))
         raise(SIGKILL);
     return ret;
 }
@@ -216,7 +237,8 @@ void *NEW(realloc)(void *ptr, size_t size)
     }
     LOADSYM(realloc);
     ret = ORIG(realloc)(ptr, size);
-    if(ret == NULL && _zz_memory && errno == ENOMEM)
+    if (_zz_memory && ((!ret && errno == ENOMEM)
+                        || (ret && memory_exceeded())))
         raise(SIGKILL);
     return ret;
 }
@@ -227,7 +249,8 @@ void *NEW(valloc)(size_t size)
     void *ret;
     LOADSYM(valloc);
     ret = ORIG(valloc)(size);
-    if(ret == NULL && _zz_memory && errno == ENOMEM)
+    if (_zz_memory && ((!ret && errno == ENOMEM)
+                        || (ret && memory_exceeded())))
         raise(SIGKILL);
     return ret;
 }
@@ -239,7 +262,8 @@ void *NEW(memalign)(size_t boundary, size_t size)
     void *ret;
     LOADSYM(memalign);
     ret = ORIG(memalign)(boundary, size);
-    if(ret == NULL && _zz_memory && errno == ENOMEM)
+    if (_zz_memory && ((!ret && errno == ENOMEM)
+                        || (ret && memory_exceeded())))
         raise(SIGKILL);
     return ret;
 }
@@ -251,7 +275,8 @@ int NEW(posix_memalign)(void **memptr, size_t alignment, size_t size)
     int ret;
     LOADSYM(posix_memalign);
     ret = ORIG(posix_memalign)(memptr, alignment, size);
-    if(ret == ENOMEM && _zz_memory)
+    if (_zz_memory && ((!ret && errno == ENOMEM)
+                        || (ret && memory_exceeded())))
         raise(SIGKILL);
     return ret;
 }

src/libzzuf/libzzuf.c

@@ -83,7 +83,7 @@ int _zz_signal = 0;
  * allowed to allocate. Its value is set by the ZZUF_MEMORY environment
  * variable.
  */
-int _zz_memory = 0;
+uint64_t _zz_memory = 0;
 
 /**
  * If set to 1, this boolean will tell libzzuf to fuzz network file
@@ -175,8 +175,8 @@ void _zz_init(void)
         _zz_signal = 1;
 
     tmp = getenv("ZZUF_MEMORY");
-    if(tmp && *tmp == '1')
-        _zz_memory = 1;
+    if(tmp)
+        _zz_memory = atoll(tmp);
 
     tmp = getenv("ZZUF_NETWORK");
     if(tmp && *tmp == '1')

src/libzzuf/libzzuf.h

@@ -20,7 +20,7 @@ extern int _zz_disabled;
 extern int _zz_debuglevel;
 extern int _zz_debugfd;
 extern int _zz_signal;
-extern int _zz_memory;
+extern uint64_t _zz_memory;
 extern int _zz_network;
 extern int _zz_autoinc;