From ab43f753618726c5da91cecf63b55835279f3e5d Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 14:49:36 -0800 Subject: [PATCH 01/11] Add hook to save crashing tests --- src/include/deepstate/DeepState.h | 9 ++++++++- src/lib/DeepState.c | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 7194376..c228107 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -346,6 +346,9 @@ extern void DeepState_SavePassingTest(void); /* Save a failing test to the output test directory. */ extern void DeepState_SaveFailingTest(void); +/* Save a crashing test to the output test directory. */ +extern void DeepState_SaveCrashingTest(void); + /* Jump buffer for returning to `DeepState_Run`. */ extern jmp_buf DeepState_ReturnToRun; @@ -469,7 +472,11 @@ static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { return status ? 1 : 0; } - /* If here, we exited abnormally, and so the test failed. */ + /* If here, we exited abnormally, and so the test failed due to a crash. */ + if (HAS_FLAG_output_test_dir) { + DeepState_SaveCrashingTest(); + } + return 1; } diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index ac61b90..6009160 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -374,6 +374,9 @@ void DeepState_SaveFailingTest(void) { printf("Saving to %s\n", FLAGS_output_test_dir); } +/* Save a crashing test to the output test directory. */ +void DeepState_SaveCrashingTest(void) {} + /* Return the first test case to run. */ struct DeepState_TestInfo *DeepState_FirstTest(void) { return DeepState_LastTestInfo; From ac7e57a8330569f841067c1234d292f2e65e877a Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 14:51:07 -0800 Subject: [PATCH 02/11] Consistently stub test case save hooks --- src/lib/DeepState.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 6009160..ec566b6 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -365,14 +365,10 @@ void DeepState_BeginDrFuzz(struct DeepState_TestInfo *test) { } /* Save a passing test to the output test directory. */ -void DeepState_SavePassingTest(void) { - -} +void DeepState_SavePassingTest(void) {} /* Save a failing test to the output test directory. */ -void DeepState_SaveFailingTest(void) { - printf("Saving to %s\n", FLAGS_output_test_dir); -} +void DeepState_SaveFailingTest(void) {} /* Save a crashing test to the output test directory. */ void DeepState_SaveCrashingTest(void) {} From 12dc706534c27dee01126768600f5408152eb06a Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 15:01:30 -0800 Subject: [PATCH 03/11] Add crashing example tests --- examples/CMakeLists.txt | 3 +++ examples/Crash.cpp | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 examples/Crash.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 28bc3e6..746b4ce 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,6 +13,9 @@ # limitations under the License. +add_executable(Crash Crash.cpp) +target_link_libraries(Crash deepstate) + add_executable(OneOf OneOf.cpp) target_link_libraries(OneOf deepstate) diff --git a/examples/Crash.cpp b/examples/Crash.cpp new file mode 100644 index 0000000..ca0e66f --- /dev/null +++ b/examples/Crash.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018 Trail of Bits, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include + +using namespace deepstate; + +DEEPSTATE_NOINLINE static unsigned segfault(unsigned x) { + if (x == 0x1234) { // Magic number for engine to discover + unsigned *p = NULL; + *p = 0xdeadbeef; // Trigger segfault here + } + + return x; +} + +TEST(Crash, SegFault) { + symbolic_unsigned x; + + segfault(x); + + ASSERT_EQ(x, x); +} + +int main(int argc, char *argv[]) { + DeepState_InitOptions(argc, argv); + return DeepState_Run(); +} From 86d0fac2064c867fd9199d8c0e42a3370871dc7f Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 15:04:29 -0800 Subject: [PATCH 04/11] Log error when test crashes --- src/include/deepstate/DeepState.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index c228107..2a0f85f 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -473,6 +473,8 @@ static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { } /* If here, we exited abnormally, and so the test failed due to a crash. */ + DeepState_LogFormat(DeepState_LogError, "Crashed: %s", test->test_name); + if (HAS_FLAG_output_test_dir) { DeepState_SaveCrashingTest(); } From 8ede1e2ddcf1b5544a7e0cd3354e11688e235aab Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 15:20:59 -0800 Subject: [PATCH 05/11] Remove stray whitespace --- bin/deepstate/common.py | 16 ++++++++-------- bin/deepstate/main_angr.py | 2 +- bin/deepstate/main_manticore.py | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index cb9a88c..f118da3 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -191,7 +191,7 @@ class DeepState(object): """Find the test case descriptors.""" tests = [] info_ea, _ = self.read_uintptr_t(self.context['apis']['LastTestInfo']) - + while info_ea: test, info_ea = self._read_test_info(info_ea) if test: @@ -219,7 +219,7 @@ class DeepState(object): self.context['log'] = [] for level in LOG_LEVEL_TO_LOGGER: self.context['stream_{}'.format(level)] = [] - + self.context['info'] = info self.log_message(LOG_LEVEL_INFO, "Running {} from {}({})".format( info.name, info.file_name, info.line_number)) @@ -294,7 +294,7 @@ class DeepState(object): val = struct.unpack(unpack_str, data)[0] else: assert val_type == int - + # TODO(pag): I am pretty sure that this is wrong for big-endian. data = struct.pack('BBBBBBBB', *val_bytes) val = struct.unpack(unpack_str, data[:struct.calcsize(unpack_str)])[0] @@ -340,7 +340,7 @@ class DeepState(object): info = self.context['info'] apis = self.context['apis'] input_length, _ = self.read_uint32_t(apis['InputIndex']) - + symbols = self.context['symbols'] # Check to see if the test case actually read too many symbols. @@ -384,7 +384,7 @@ class DeepState(object): """Notify the symbolic executor that this test has been abandoned due to some critical error and stop executing the current state.""" self.context['abandoned'] = True - + def api_min_uint(self, arg): """Implements the `DeepState_MinUInt` API function, which returns the minimum satisfiable value for `arg`.""" @@ -417,7 +417,7 @@ class DeepState(object): self.abandon_test() else: return - + constraint = arg != 0 if not self.add_constraint(constraint): expr, _ = self.read_c_string(expr_ea, concretize=False) @@ -500,7 +500,7 @@ class DeepState(object): ea = self.concretize(ea, constrain=True) assert level in LOG_LEVEL_TO_LOGGER self.log_message(level, self.read_c_string(ea, concretize=False)[0]) - + if level == LOG_LEVEL_FATAL: self.api_fail() elif level == LOG_LEVEL_ERROR: @@ -551,7 +551,7 @@ class DeepState(object): str_ea = self.concretize(str_ea, constrain=True) format_str = self.read_c_string(format_ea)[0] print_str = self.read_c_string(str_ea, concretize=False)[0] - + stream_id = 'stream_{}'.format(level) stream = list(self.context[stream_id]) stream.append((str, format_str, None, print_str)) diff --git a/bin/deepstate/main_angr.py b/bin/deepstate/main_angr.py index 30bf579..bcac38c 100644 --- a/bin/deepstate/main_angr.py +++ b/bin/deepstate/main_angr.py @@ -265,7 +265,7 @@ def do_run_test(project, test, apis, run_state): mc = DeepAngr(state=test_state) mc.begin_test(test) del mc - + errored = [] test_manager = angr.SimulationManager( project=project, diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index d53eadf..05b3f68 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -272,7 +272,7 @@ def do_run_test(state, apis, test): mc = DeepManticore(state) mc.begin_test(test) del mc - + m.add_hook(apis['IsSymbolicUInt'], hook(hook_IsSymbolicUInt)) m.add_hook(apis['ConcretizeData'], hook(hook_ConcretizeData)) m.add_hook(apis['ConcretizeCStr'], hook(hook_ConcretizeCStr)) @@ -308,7 +308,7 @@ def run_tests(args, state, apis): results = [] mc = DeepManticore(state) tests = mc.find_test_cases() - + L.info("Running {} tests across {} workers".format( len(tests), args.num_workers)) @@ -334,7 +334,7 @@ def main(): m.verbosity(1) - # Hack to get around current broken _get_symbol_address + # Hack to get around current broken _get_symbol_address m._binary_type = 'not elf' m._binary_obj = m._initial_state.platform.elf @@ -352,7 +352,7 @@ def main(): if not ea_of_api_table: L.critical("Could not find API table in binary `{}`".format(args.binary)) return 1 - + apis = mc.read_api_table(ea_of_api_table) del mc m.add_hook(setup_ea, lambda state: run_tests(args, state, apis)) From 23af5b562d8f20f3e6328e560797f7d950fd97da Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Wed, 14 Feb 2018 12:03:45 -0800 Subject: [PATCH 06/11] Add `DeepState_Crash()` hook --- src/include/deepstate/DeepState.h | 4 ++++ src/lib/DeepState.c | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 2a0f85f..06ef3e3 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -184,6 +184,9 @@ extern void _DeepState_Assume(int expr, const char *expr_str, const char *file, DEEPSTATE_NORETURN extern void DeepState_Abandon(const char *reason); +/* Mark this test as having crashed. */ +extern void DeepState_Crash(void); + DEEPSTATE_NORETURN extern void DeepState_Fail(void); @@ -473,6 +476,7 @@ static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { } /* If here, we exited abnormally, and so the test failed due to a crash. */ + DeepState_Crash(); DeepState_LogFormat(DeepState_LogError, "Crashed: %s", test->test_name); if (HAS_FLAG_output_test_dir) { diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index ec566b6..14a428a 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -55,6 +55,11 @@ void DeepState_Abandon(const char *reason) { longjmp(DeepState_ReturnToRun, 1); } +/* Mark this test as having crashed. */ +void DeepState_Crash(void) { + DeepState_TestFailed = 1; +} + /* Mark this test as failing. */ DEEPSTATE_NORETURN void DeepState_Fail(void) { @@ -261,6 +266,7 @@ const struct DeepState_IndexEntry DeepState_API[] = { /* Control-flow during the test. */ {"Pass", (void *) DeepState_Pass}, + {"Crash", (void *) DeepState_Crash}, {"Fail", (void *) DeepState_Fail}, {"SoftFail", (void *) DeepState_SoftFail}, {"Abandon", (void *) DeepState_Abandon}, From 7fbb96677792c733302c6e2c03b71daf07306fda Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Wed, 14 Feb 2018 12:05:07 -0800 Subject: [PATCH 07/11] Detect, report crashes in `deepstate-angr` --- bin/deepstate/common.py | 16 ++++++++++++++++ bin/deepstate/main_angr.py | 12 ++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index f118da3..61b7eef 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -215,6 +215,7 @@ class DeepState(object): def begin_test(self, info): """Begin processing the test associated with `info`.""" self.context['failed'] = False + self.context['crashed'] = False self.context['abandoned'] = False self.context['log'] = [] for level in LOG_LEVEL_TO_LOGGER: @@ -324,6 +325,8 @@ class DeepState(object): if self.context['failed']: test_name += ".fail" + elif self.context['crashed']: + test_name += ".crash" else: test_name += ".pass" @@ -375,6 +378,11 @@ class DeepState(object): executing the current state.""" pass + def crash_test(self): + """Notify the symbolic executor that this test has crashed and stop + executing the current state.""" + self.context['crashed'] = True + def fail_test(self): """Notify the symbolic executor that this test has failed and stop executing the current state.""" @@ -469,6 +477,14 @@ class DeepState(object): self.log_message(LOG_LEVEL_INFO, "Passed: {}".format(info.name)) self.pass_test() + def api_crash(self): + """Implements the `DeepState_Crash` API function, which marks this test as + having failed, and stops further execution.""" + self.context['crashed'] = True + info = self.context['info'] + self.log_message(LOG_LEVEL_ERROR, "Crashed: {}".format(info.name)) + self.crash_test() + def api_fail(self): """Implements the `DeepState_Fail` API function, which marks this test as having failed, and stops further execution.""" diff --git a/bin/deepstate/main_angr.py b/bin/deepstate/main_angr.py index bcac38c..d7c8097 100644 --- a/bin/deepstate/main_angr.py +++ b/bin/deepstate/main_angr.py @@ -165,6 +165,12 @@ class Pass(angr.SimProcedure): DeepAngr(procedure=self).api_pass() +class Crash(angr.SimProcedure): + """Implements DeepState_Crash, which notifies us of a crashing test.""" + def run(self): + DeepAngr(procedure=self).api_crash() + + class Fail(angr.SimProcedure): """Implements DeepState_Fail, which notifies us of a failing test.""" def run(self): @@ -281,8 +287,9 @@ def do_run_test(project, test, apis, run_state): DeepAngr(state=state).report() for error in test_manager.errored: - print "Error", error.error - error.debug() + da = DeepAngr(state=error.state) + da.crash_test() + da.report() def run_test(project, test, apis, run_state): """Symbolically executes a single test function.""" @@ -374,6 +381,7 @@ def main(): hook_function(project, apis['MaxUInt'], MaxUInt) hook_function(project, apis['Assume'], Assume) hook_function(project, apis['Pass'], Pass) + hook_function(project, apis['Crash'], Crash) hook_function(project, apis['Fail'], Fail) hook_function(project, apis['Abandon'], Abandon) hook_function(project, apis['SoftFail'], SoftFail) From 0b82d4e4099be912954a3373b23584aa6036878a Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Wed, 14 Feb 2018 12:30:14 -0800 Subject: [PATCH 08/11] Run saved .crash test cases in native harness --- src/include/deepstate/DeepState.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 06ef3e3..f917928 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -364,8 +364,17 @@ static bool IsTestCaseFile(const char *name) { return false; } - if (strcmp(suffix, ".pass") == 0 || strcmp(suffix, ".fail") == 0) { - return true; + const char *extensions[] = { + ".pass", + ".fail", + ".crash", + }; + const size_t ext_count = sizeof(extensions) / sizeof(char *); + + for (size_t i = 0; i < ext_count; i++) { + if (!strcmp(suffix, extensions[i])) { + return true; + } } return false; From c1b91432fd0b9d73b046d1915b220a375d48bc17 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Wed, 14 Feb 2018 14:25:10 -0800 Subject: [PATCH 09/11] Fix executor docstrings --- bin/deepstate/common.py | 2 +- bin/deepstate/main_manticore.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index 61b7eef..9a2df0d 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -479,7 +479,7 @@ class DeepState(object): def api_crash(self): """Implements the `DeepState_Crash` API function, which marks this test as - having failed, and stops further execution.""" + having crashed, and stops further execution.""" self.context['crashed'] = True info = self.context['info'] self.log_message(LOG_LEVEL_ERROR, "Crashed: {}".format(info.name)) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index 05b3f68..88ec543 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -186,7 +186,7 @@ def hook_Pass(state): def hook_Fail(state): - """Implements DeepState_Fail, which notifies us of a passing test.""" + """Implements DeepState_Fail, which notifies us of a failing test.""" DeepManticore(state).api_fail() From 4814e8184be7344416f8b3352d30da224f143aec Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Wed, 14 Feb 2018 17:21:17 -0800 Subject: [PATCH 10/11] Flag unknown Manticore state terminations as crashes This is an over-approximation, which we will tighten later. --- bin/deepstate/main_manticore.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index 88ec543..b2aa623 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -130,6 +130,10 @@ class DeepManticore(DeepState): super(DeepManticore, self).pass_test() raise TerminateState(OUR_TERMINATION_REASON, testcase=False) + def crash_test(self): + super(DeepManticore, self).crash_test() + raise TerminateState(OUR_TERMINATION_REASON, testcase=False) + def fail_test(self): super(DeepManticore, self).fail_test() raise TerminateState(OUR_TERMINATION_REASON, testcase=False) @@ -184,6 +188,9 @@ def hook_Pass(state): """Implements DeepState_Pass, which notifies us of a passing test.""" DeepManticore(state).api_pass() +def hook_Crash(state): + """Implements DeepState_Crash, which notifies us of a crashing test.""" + DeepManticore(state).api_crash() def hook_Fail(state): """Implements DeepState_Fail, which notifies us of a failing test.""" @@ -238,11 +245,13 @@ def hook(func): def done_test(_, state, state_id, reason): """Called when a state is terminated.""" - if OUR_TERMINATION_REASON not in reason: - L.error("State {} terminated for unknown reason: {}".format( - state_id, reason)) - return mc = DeepManticore(state) + + if OUR_TERMINATION_REASON not in reason: + L.info("State {} terminated for unknown reason, treating as crash: {}".format( + state_id, reason)) + super(DeepManticore, mc).crash_test() + mc.report() @@ -280,6 +289,7 @@ def do_run_test(state, apis, test): m.add_hook(apis['MaxUInt'], hook(hook_MaxUInt)) m.add_hook(apis['Assume'], hook(hook_Assume)) m.add_hook(apis['Pass'], hook(hook_Pass)) + m.add_hook(apis['Crash'], hook(hook_Crash)) m.add_hook(apis['Fail'], hook(hook_Fail)) m.add_hook(apis['SoftFail'], hook(hook_SoftFail)) m.add_hook(apis['Abandon'], hook(hook_Abandon)) From 226d0f5513855779939173582f3f454ca340692c Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Thu, 15 Feb 2018 12:43:12 -0800 Subject: [PATCH 11/11] Add enum for test run results and exit codes --- src/include/deepstate/DeepState.h | 67 ++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index f917928..dd9f42f 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -180,6 +180,16 @@ extern void _DeepState_Assume(int expr, const char *expr_str, const char *file, #define DeepState_Assume(x) _DeepState_Assume(!!(x), #x, __FILE__, __LINE__) +/* Result of a single forked test run. + * + * Will be passed to the parent process as an exit code. */ +enum DeepState_TestRunResult { + DeepState_TestRunPass = 0, + DeepState_TestRunFail = 1, + DeepState_TestRunCrash = 2, + DeepState_TestRunAbandon = 3, +}; + /* Abandon this test. We've hit some kind of internal problem. */ DEEPSTATE_NORETURN extern void DeepState_Abandon(const char *reason); @@ -434,7 +444,7 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { #endif /* __cplusplus */ test->test_func(); /* Run the test function. */ - exit(0); + exit(DeepState_TestRunPass); #if defined(__cplusplus) && defined(__cpp_exceptions) } catch(...) { @@ -448,13 +458,13 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { if (HAS_FLAG_output_test_dir) { DeepState_SaveFailingTest(); } - exit(1); + exit(DeepState_TestRunFail); /* The test was abandoned. We may have gotten soft failures before * abandoning, so we prefer to catch those first. */ } else if (DeepState_CatchAbandoned()) { DeepState_LogFormat(DeepState_LogFatal, "Abandoned: %s", test->test_name); - exit(1); + exit(DeepState_TestRunAbandon); /* The test passed. */ } else { @@ -462,14 +472,13 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { if (HAS_FLAG_output_test_dir) { DeepState_SavePassingTest(); } - exit(0); + exit(DeepState_TestRunPass); } } -/* Fork and run `test`. - * - * Returns 1 if the test failed, 0 otherwise. */ -static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { +/* Fork and run `test`. */ +static enum DeepState_TestRunResult +DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { pid_t test_pid = fork(); if (!test_pid) { DeepState_RunTest(test); @@ -480,25 +489,19 @@ static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { /* If we exited normally, the status code tells us if the test passed. */ if (WIFEXITED(wstatus)) { uint8_t status = WEXITSTATUS(wstatus); - - return status ? 1 : 0; + return (enum DeepState_TestRunResult) status; } - /* If here, we exited abnormally, and so the test failed due to a crash. */ - DeepState_Crash(); - DeepState_LogFormat(DeepState_LogError, "Crashed: %s", test->test_name); - - if (HAS_FLAG_output_test_dir) { - DeepState_SaveCrashingTest(); - } - - return 1; + /* If here, we exited abnormally but didn't catch it in the signal + * handler, and thus the test failed due to a crash. */ + return DeepState_TestRunCrash; } /* Run a single saved test case with input initialized from the file * `name` in directory `dir`. */ -static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, - const char *dir, const char *name) { +static enum DeepState_TestRunResult +DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, const char *dir, + const char *name) { size_t path_len = 2 + sizeof(char) * (strlen(dir) + strlen(name)); char *path = (char *) malloc(path_len); if (path == NULL) { @@ -512,7 +515,19 @@ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, DeepState_Begin(test); - return DeepState_ForkAndRunTest(test); + enum DeepState_TestRunResult result = DeepState_ForkAndRunTest(test); + + if (result == DeepState_TestRunCrash) { + DeepState_LogFormat(DeepState_LogError, "Crashed: %s", test->test_name); + + if (HAS_FLAG_output_test_dir) { + DeepState_SaveCrashingTest(); + } + + DeepState_Crash(); + } + + return result; } /* Run tests with saved input from `FLAGS_input_test_dir`. @@ -552,8 +567,12 @@ static int DeepState_RunSavedTestCases(void) { /* Read generated test cases and run a test for each file found. */ while ((dp = readdir(dir_fd)) != NULL) { if (IsTestCaseFile(dp->d_name)) { - num_failed_tests += DeepState_DoRunSavedTestCase(test, test_case_dir, - dp->d_name); + enum DeepState_TestRunResult result = + DeepState_DoRunSavedTestCase(test, test_case_dir, dp->d_name); + + if (result != DeepState_TestRunPass) { + num_failed_tests++; + } } } closedir(dir_fd);