Merge pull request #30 from trailofbits/save-crashes

Save crashes

bin/deepstate/common.py

@@ -191,7 +191,7 @@ class DeepState(object):
     """Find the test case descriptors."""
     tests = []
     info_ea, _ = self.read_uintptr_t(self.context['apis']['LastTestInfo'])
-    
+
     while info_ea:
       test, info_ea = self._read_test_info(info_ea)
       if test:
@@ -215,11 +215,12 @@ class DeepState(object):
   def begin_test(self, info):
     """Begin processing the test associated with `info`."""
     self.context['failed'] = False
+    self.context['crashed'] = False
     self.context['abandoned'] = False
     self.context['log'] = []
     for level in LOG_LEVEL_TO_LOGGER:
       self.context['stream_{}'.format(level)] = []
-    
+
     self.context['info'] = info
     self.log_message(LOG_LEVEL_INFO, "Running {} from {}({})".format(
         info.name, info.file_name, info.line_number))
@@ -294,7 +295,7 @@ class DeepState(object):
         val = struct.unpack(unpack_str, data)[0]
       else:
         assert val_type == int
-        
+
         # TODO(pag): I am pretty sure that this is wrong for big-endian.
         data = struct.pack('BBBBBBBB', *val_bytes)
         val = struct.unpack(unpack_str, data[:struct.calcsize(unpack_str)])[0]
@@ -324,6 +325,8 @@ class DeepState(object):
 
     if self.context['failed']:
       test_name += ".fail"
+    elif self.context['crashed']:
+      test_name += ".crash"
     else:
       test_name += ".pass"
 
@@ -340,7 +343,7 @@ class DeepState(object):
     info = self.context['info']
     apis = self.context['apis']
     input_length, _ = self.read_uint32_t(apis['InputIndex'])
-    
+
     symbols = self.context['symbols']
 
     # Check to see if the test case actually read too many symbols.
@@ -375,6 +378,11 @@ class DeepState(object):
     executing the current state."""
     pass
 
+  def crash_test(self):
+    """Notify the symbolic executor that this test has crashed and stop
+    executing the current state."""
+    self.context['crashed'] = True
+
   def fail_test(self):
     """Notify the symbolic executor that this test has failed and stop
     executing the current state."""
@@ -384,7 +392,7 @@ class DeepState(object):
     """Notify the symbolic executor that this test has been abandoned due to
     some critical error and stop executing the current state."""
     self.context['abandoned'] = True
-  
+
   def api_min_uint(self, arg):
     """Implements the `DeepState_MinUInt` API function, which returns the
     minimum satisfiable value for `arg`."""
@@ -417,7 +425,7 @@ class DeepState(object):
         self.abandon_test()
       else:
         return
-    
+
     constraint = arg != 0
     if not self.add_constraint(constraint):
       expr, _ = self.read_c_string(expr_ea, concretize=False)
@@ -469,6 +477,14 @@ class DeepState(object):
       self.log_message(LOG_LEVEL_INFO, "Passed: {}".format(info.name))
       self.pass_test()
 
+  def api_crash(self):
+    """Implements the `DeepState_Crash` API function, which marks this test as
+    having crashed, and stops further execution."""
+    self.context['crashed'] = True
+    info = self.context['info']
+    self.log_message(LOG_LEVEL_ERROR, "Crashed: {}".format(info.name))
+    self.crash_test()
+
   def api_fail(self):
     """Implements the `DeepState_Fail` API function, which marks this test as
     having failed, and stops further execution."""
@@ -500,7 +516,7 @@ class DeepState(object):
     ea = self.concretize(ea, constrain=True)
     assert level in LOG_LEVEL_TO_LOGGER
     self.log_message(level, self.read_c_string(ea, concretize=False)[0])
-    
+
     if level == LOG_LEVEL_FATAL:
       self.api_fail()
     elif level == LOG_LEVEL_ERROR:
@@ -551,7 +567,7 @@ class DeepState(object):
     str_ea = self.concretize(str_ea, constrain=True)
     format_str = self.read_c_string(format_ea)[0]
     print_str = self.read_c_string(str_ea, concretize=False)[0]
-    
+
     stream_id = 'stream_{}'.format(level)
     stream = list(self.context[stream_id])
     stream.append((str, format_str, None, print_str))

bin/deepstate/main_angr.py

@@ -165,6 +165,12 @@ class Pass(angr.SimProcedure):
     DeepAngr(procedure=self).api_pass()
 
 
+class Crash(angr.SimProcedure):
+  """Implements DeepState_Crash, which notifies us of a crashing test."""
+  def run(self):
+    DeepAngr(procedure=self).api_crash()
+
+
 class Fail(angr.SimProcedure):
   """Implements DeepState_Fail, which notifies us of a failing test."""
   def run(self):
@@ -265,7 +271,7 @@ def do_run_test(project, test, apis, run_state):
   mc = DeepAngr(state=test_state)
   mc.begin_test(test)
   del mc
-  
+
   errored = []
   test_manager = angr.SimulationManager(
       project=project,
@@ -281,8 +287,9 @@ def do_run_test(project, test, apis, run_state):
     DeepAngr(state=state).report()
 
   for error in test_manager.errored:
-    print "Error", error.error
-    error.debug()
+    da = DeepAngr(state=error.state)
+    da.crash_test()
+    da.report()
 
 def run_test(project, test, apis, run_state):
   """Symbolically executes a single test function."""
@@ -374,6 +381,7 @@ def main():
   hook_function(project, apis['MaxUInt'], MaxUInt)
   hook_function(project, apis['Assume'], Assume)
   hook_function(project, apis['Pass'], Pass)
+  hook_function(project, apis['Crash'], Crash)
   hook_function(project, apis['Fail'], Fail)
   hook_function(project, apis['Abandon'], Abandon)
   hook_function(project, apis['SoftFail'], SoftFail)

bin/deepstate/main_manticore.py

@@ -130,6 +130,10 @@ class DeepManticore(DeepState):
     super(DeepManticore, self).pass_test()
     raise TerminateState(OUR_TERMINATION_REASON, testcase=False)
 
+  def crash_test(self):
+    super(DeepManticore, self).crash_test()
+    raise TerminateState(OUR_TERMINATION_REASON, testcase=False)
+
   def fail_test(self):
     super(DeepManticore, self).fail_test()
     raise TerminateState(OUR_TERMINATION_REASON, testcase=False)
@@ -184,9 +188,12 @@ def hook_Pass(state):
   """Implements DeepState_Pass, which notifies us of a passing test."""
   DeepManticore(state).api_pass()
 
+def hook_Crash(state):
+  """Implements DeepState_Crash, which notifies us of a crashing test."""
+  DeepManticore(state).api_crash()
 
 def hook_Fail(state):
-  """Implements DeepState_Fail, which notifies us of a passing test."""
+  """Implements DeepState_Fail, which notifies us of a failing test."""
   DeepManticore(state).api_fail()
 
 
@@ -238,11 +245,13 @@ def hook(func):
 
 def done_test(_, state, state_id, reason):
   """Called when a state is terminated."""
-  if OUR_TERMINATION_REASON not in reason:
-    L.error("State {} terminated for unknown reason: {}".format(
-        state_id, reason))
-    return
   mc = DeepManticore(state)
+
+  if OUR_TERMINATION_REASON not in reason:
+    L.info("State {} terminated for unknown reason, treating as crash: {}".format(
+      state_id, reason))
+    super(DeepManticore, mc).crash_test()
+
   mc.report()
 
 
@@ -272,7 +281,7 @@ def do_run_test(state, apis, test):
   mc = DeepManticore(state)
   mc.begin_test(test)
   del mc
-  
+
   m.add_hook(apis['IsSymbolicUInt'], hook(hook_IsSymbolicUInt))
   m.add_hook(apis['ConcretizeData'], hook(hook_ConcretizeData))
   m.add_hook(apis['ConcretizeCStr'], hook(hook_ConcretizeCStr))
@@ -280,6 +289,7 @@ def do_run_test(state, apis, test):
   m.add_hook(apis['MaxUInt'], hook(hook_MaxUInt))
   m.add_hook(apis['Assume'], hook(hook_Assume))
   m.add_hook(apis['Pass'], hook(hook_Pass))
+  m.add_hook(apis['Crash'], hook(hook_Crash))
   m.add_hook(apis['Fail'], hook(hook_Fail))
   m.add_hook(apis['SoftFail'], hook(hook_SoftFail))
   m.add_hook(apis['Abandon'], hook(hook_Abandon))
@@ -308,7 +318,7 @@ def run_tests(args, state, apis):
   results = []
   mc = DeepManticore(state)
   tests = mc.find_test_cases()
-  
+
   L.info("Running {} tests across {} workers".format(
       len(tests), args.num_workers))
 
@@ -334,7 +344,7 @@ def main():
 
   m.verbosity(1)
 
-  # Hack to get around current broken _get_symbol_address 
+  # Hack to get around current broken _get_symbol_address
   m._binary_type = 'not elf'
   m._binary_obj = m._initial_state.platform.elf
 
@@ -352,7 +362,7 @@ def main():
   if not ea_of_api_table:
     L.critical("Could not find API table in binary `{}`".format(args.binary))
     return 1
-    
+
   apis = mc.read_api_table(ea_of_api_table)
   del mc
   m.add_hook(setup_ea, lambda state: run_tests(args, state, apis))

examples/CMakeLists.txt

@@ -13,6 +13,9 @@
 # limitations under the License.
 
 
+add_executable(Crash Crash.cpp)
+target_link_libraries(Crash deepstate)
+
 add_executable(OneOf OneOf.cpp)
 target_link_libraries(OneOf deepstate)
 

examples/Crash.cpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2018 Trail of Bits, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <deepstate/DeepState.hpp>
+
+using namespace deepstate;
+
+DEEPSTATE_NOINLINE static unsigned segfault(unsigned x) {
+  if (x == 0x1234) {  // Magic number for engine to discover
+    unsigned *p = NULL;
+    *p = 0xdeadbeef;  // Trigger segfault here
+  }
+
+  return x;
+}
+
+TEST(Crash, SegFault) {
+  symbolic_unsigned x;
+
+  segfault(x);
+
+  ASSERT_EQ(x, x);
+}
+
+int main(int argc, char *argv[]) {
+  DeepState_InitOptions(argc, argv);
+  return DeepState_Run();
+}

src/include/deepstate/DeepState.h

@@ -180,10 +180,23 @@ extern void _DeepState_Assume(int expr, const char *expr_str, const char *file,
 
 #define DeepState_Assume(x) _DeepState_Assume(!!(x), #x, __FILE__, __LINE__)
 
+/* Result of a single forked test run.
+ *
+ * Will be passed to the parent process as an exit code. */
+enum DeepState_TestRunResult {
+  DeepState_TestRunPass = 0,
+  DeepState_TestRunFail = 1,
+  DeepState_TestRunCrash = 2,
+  DeepState_TestRunAbandon = 3,
+};
+
 /* Abandon this test. We've hit some kind of internal problem. */
 DEEPSTATE_NORETURN
 extern void DeepState_Abandon(const char *reason);
 
+/* Mark this test as having crashed. */
+extern void DeepState_Crash(void);
+
 DEEPSTATE_NORETURN
 extern void DeepState_Fail(void);
 
@@ -346,6 +359,9 @@ extern void DeepState_SavePassingTest(void);
 /* Save a failing test to the output test directory. */
 extern void DeepState_SaveFailingTest(void);
 
+/* Save a crashing test to the output test directory. */
+extern void DeepState_SaveCrashingTest(void);
+
 /* Jump buffer for returning to `DeepState_Run`. */
 extern jmp_buf DeepState_ReturnToRun;
 
@@ -358,8 +374,17 @@ static bool IsTestCaseFile(const char *name) {
     return false;
   }
 
-  if (strcmp(suffix, ".pass") == 0 || strcmp(suffix, ".fail") == 0) {
-    return true;
+  const char *extensions[] = {
+    ".pass",
+    ".fail",
+    ".crash",
+  };
+  const size_t ext_count = sizeof(extensions) / sizeof(char *);
+
+  for (size_t i = 0; i < ext_count; i++) {
+    if (!strcmp(suffix, extensions[i])) {
+      return true;
+    }
   }
 
   return false;
@@ -419,7 +444,7 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) {
 #endif  /* __cplusplus */
 
       test->test_func();  /* Run the test function. */
-      exit(0);
+      exit(DeepState_TestRunPass);
 
 #if defined(__cplusplus) && defined(__cpp_exceptions)
     } catch(...) {
@@ -433,13 +458,13 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) {
     if (HAS_FLAG_output_test_dir) {
       DeepState_SaveFailingTest();
     }
-    exit(1);
+    exit(DeepState_TestRunFail);
 
     /* The test was abandoned. We may have gotten soft failures before
      * abandoning, so we prefer to catch those first. */
   } else if (DeepState_CatchAbandoned()) {
     DeepState_LogFormat(DeepState_LogFatal, "Abandoned: %s", test->test_name);
-    exit(1);
+    exit(DeepState_TestRunAbandon);
 
     /* The test passed. */
   } else {
@@ -447,14 +472,13 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) {
     if (HAS_FLAG_output_test_dir) {
       DeepState_SavePassingTest();
     }
-    exit(0);
+    exit(DeepState_TestRunPass);
   }
 }
 
-/* Fork and run `test`.
- *
- * Returns 1 if the test failed, 0 otherwise. */
-static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) {
+/* Fork and run `test`. */
+static enum DeepState_TestRunResult
+DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) {
   pid_t test_pid = fork();
   if (!test_pid) {
     DeepState_RunTest(test);
@@ -465,18 +489,19 @@ static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) {
   /* If we exited normally, the status code tells us if the test passed. */
   if (WIFEXITED(wstatus)) {
     uint8_t status = WEXITSTATUS(wstatus);
-
-    return status ? 1 : 0;
+    return (enum DeepState_TestRunResult) status;
   }
 
-  /* If here, we exited abnormally, and so the test failed. */
-  return 1;
+  /* If here, we exited abnormally but didn't catch it in the signal
+   * handler, and thus the test failed due to a crash. */
+  return DeepState_TestRunCrash;
 }
 
 /* Run a single saved test case with input initialized from the file
  * `name` in directory `dir`. */
-static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test,
-                                        const char *dir, const char *name) {
+static enum DeepState_TestRunResult
+DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, const char *dir,
+                             const char *name) {
   size_t path_len = 2 + sizeof(char) * (strlen(dir) + strlen(name));
   char *path = (char *) malloc(path_len);
   if (path == NULL) {
@@ -490,7 +515,19 @@ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test,
 
   DeepState_Begin(test);
 
-  return DeepState_ForkAndRunTest(test);
+  enum DeepState_TestRunResult result = DeepState_ForkAndRunTest(test);
+
+  if (result == DeepState_TestRunCrash) {
+    DeepState_LogFormat(DeepState_LogError, "Crashed: %s", test->test_name);
+
+    if (HAS_FLAG_output_test_dir) {
+      DeepState_SaveCrashingTest();
+    }
+
+    DeepState_Crash();
+  }
+
+  return result;
 }
 
 /* Run tests with saved input from `FLAGS_input_test_dir`.
@@ -530,8 +567,12 @@ static int DeepState_RunSavedTestCases(void) {
     /* Read generated test cases and run a test for each file found. */
     while ((dp = readdir(dir_fd)) != NULL) {
       if (IsTestCaseFile(dp->d_name)) {
-        num_failed_tests += DeepState_DoRunSavedTestCase(test, test_case_dir,
-                                                         dp->d_name);
+        enum DeepState_TestRunResult result =
+          DeepState_DoRunSavedTestCase(test, test_case_dir, dp->d_name);
+
+        if (result != DeepState_TestRunPass) {
+          num_failed_tests++;
+        }
       }
     }
     closedir(dir_fd);

src/lib/DeepState.c

@@ -55,6 +55,11 @@ void DeepState_Abandon(const char *reason) {
   longjmp(DeepState_ReturnToRun, 1);
 }
 
+/* Mark this test as having crashed. */
+void DeepState_Crash(void) {
+  DeepState_TestFailed = 1;
+}
+
 /* Mark this test as failing. */
 DEEPSTATE_NORETURN
 void DeepState_Fail(void) {
@@ -261,6 +266,7 @@ const struct DeepState_IndexEntry DeepState_API[] = {
 
   /* Control-flow during the test. */
   {"Pass",            (void *) DeepState_Pass},
+  {"Crash",           (void *) DeepState_Crash},
   {"Fail",            (void *) DeepState_Fail},
   {"SoftFail",        (void *) DeepState_SoftFail},
   {"Abandon",         (void *) DeepState_Abandon},
@@ -365,14 +371,13 @@ void DeepState_BeginDrFuzz(struct DeepState_TestInfo *test) {
 }
 
 /* Save a passing test to the output test directory. */
-void DeepState_SavePassingTest(void) {
-
-}
+void DeepState_SavePassingTest(void) {}
 
 /* Save a failing test to the output test directory. */
-void DeepState_SaveFailingTest(void) {
-  printf("Saving to %s\n", FLAGS_output_test_dir);
-}
+void DeepState_SaveFailingTest(void) {}
+
+/* Save a crashing test to the output test directory. */
+void DeepState_SaveCrashingTest(void) {}
 
 /* Return the first test case to run. */
 struct DeepState_TestInfo *DeepState_FirstTest(void) {