From 0a746ca07864ba299cfc6c8382e36a990214b6ac Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Mon, 19 Feb 2018 11:43:59 -0800 Subject: [PATCH 01/14] Factor out `deepstate-angr` API hook setup, unit test exec --- bin/deepstate/main_angr.py | 112 +++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 22 deletions(-) diff --git a/bin/deepstate/main_angr.py b/bin/deepstate/main_angr.py index d7c8097..28a651c 100644 --- a/bin/deepstate/main_angr.py +++ b/bin/deepstate/main_angr.py @@ -314,31 +314,42 @@ def find_symbol_ea(project, name): return 0 -def main(): - """Run DeepState.""" - args = DeepAngr.parse_args() - try: - project = angr.Project( - args.binary, - use_sim_procedures=True, - translation_cache=True, - support_selfmodifying_code=False, - auto_load_libs=True, - exclude_sim_procedures_list=['printf', '__printf_chk', - 'vprintf', '__vprintf_chk', - 'fprintf', '__fprintf_chk', - 'vfprintf', '__vfprintf_chk', - 'puts', 'abort', '__assert_fail', - '__stack_chk_fail']) - except Exception as e: - L.critical("Cannot create Angr instance on binary {}: {}".format( - args.binary, e)) +def hook_apis(project, run_state): + # Read the API table, which will tell us about the location of various + # symbols. Technically we can look these up with the `labels.lookup` API, + # but we have the API table for Manticore-compatibility, so we may as well + # use it. + ea_of_api_table = find_symbol_ea(project, 'DeepState_API') + if not ea_of_api_table: + L.critical("Could not find API table in binary `{}`".format(args.binary)) return 1 - setup_ea = find_symbol_ea(project, 'DeepState_Setup') - if not setup_ea: - L.critical("Cannot find symbol `DeepState_Setup` in binary `{}`".format( + mc = DeepAngr(state=run_state) + apis = mc.read_api_table(ea_of_api_table) + + # Hook various functions. + hook_function(project, apis['IsSymbolicUInt'], IsSymbolicUInt) + hook_function(project, apis['ConcretizeData'], ConcretizeData) + hook_function(project, apis['ConcretizeCStr'], ConcretizeCStr) + hook_function(project, apis['MinUInt'], MinUInt) + hook_function(project, apis['MaxUInt'], MaxUInt) + hook_function(project, apis['Assume'], Assume) + hook_function(project, apis['Pass'], Pass) + hook_function(project, apis['Crash'], Crash) + hook_function(project, apis['Fail'], Fail) + hook_function(project, apis['Abandon'], Abandon) + hook_function(project, apis['SoftFail'], SoftFail) + hook_function(project, apis['Log'], Log) + hook_function(project, apis['StreamInt'], StreamInt) + hook_function(project, apis['StreamFloat'], StreamFloat) + hook_function(project, apis['StreamString'], StreamString) + hook_function(project, apis['ClearStream'], ClearStream) + hook_function(project, apis['LogStream'], LogStream) + + return mc, apis + + args.binary)) return 1 @@ -392,6 +403,37 @@ def main(): hook_function(project, apis['ClearStream'], ClearStream) hook_function(project, apis['LogStream'], LogStream) + + +def main_unit_test(args, project): + setup_ea = find_symbol_ea(project, 'DeepState_Setup') + if not setup_ea: + L.critical("Cannot find symbol `DeepState_Setup` in binary `{}`".format( + args.binary)) + return 1 + + entry_state = project.factory.entry_state( + add_options={angr.options.ZERO_FILL_UNCONSTRAINED_MEMORY, + angr.options.STRICT_PAGE_ACCESS}) + + addr_size_bits = entry_state.arch.bits + + # Concretely execute up until `DeepState_Setup`. + concrete_manager = angr.SimulationManager( + project=project, + active_states=[entry_state]) + concrete_manager.explore(find=setup_ea) + + try: + run_state = concrete_manager.found[0] + except: + L.critical("Execution never hit `DeepState_Setup` in binary `{}`".format( + args.binary)) + return 1 + + # Hook the DeepState API functions. + mc, apis = hook_apis(project, run_state) + # Find the test cases that we want to run. tests = mc.find_test_cases() del mc @@ -414,5 +456,31 @@ def main(): return 0 + +def main(): + """Run DeepState.""" + args = DeepAngr.parse_args() + + try: + project = angr.Project( + args.binary, + use_sim_procedures=True, + translation_cache=True, + support_selfmodifying_code=False, + auto_load_libs=True, + exclude_sim_procedures_list=['printf', '__printf_chk', + 'vprintf', '__vprintf_chk', + 'fprintf', '__fprintf_chk', + 'vfprintf', '__vfprintf_chk', + 'puts', 'abort', '__assert_fail', + '__stack_chk_fail']) + except Exception as e: + L.critical("Cannot create Angr instance on binary {}: {}".format( + args.binary, e)) + return 1 + + return main_unit_test(args, project) + + if "__main__" == __name__: exit(main()) From 45fcfe09217a624657c784666d6441d017cdef02 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Mon, 19 Feb 2018 11:45:57 -0800 Subject: [PATCH 02/14] Add `DeepState_TakeOver()` interface --- src/include/deepstate/DeepState.h | 2 ++ src/lib/DeepState.c | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index dd9f42f..121327f 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -309,6 +309,8 @@ struct DeepState_TestInfo { /* Pointer to the last registered `TestInfo` structure. */ extern struct DeepState_TestInfo *DeepState_LastTestInfo; +extern int DeepState_TakeOver(void); + /* Defines the entrypoint of a test case. This creates a data structure that * contains the information about the test, and then creates an initializer * function that runs before `main` that registers the test entrypoint with diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 14a428a..05b4b79 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -364,6 +364,19 @@ void DrMemFuzzFunc(volatile uint8_t *buff, size_t size) { } } +int DeepState_TakeOver(void) { + struct DeepState_TestInfo test = { + .prev = NULL, + .test_func = NULL, + .test_name = "<__TAKE_OVER_TEST>", + .file_name = "<__TAKE_OVER_FILE>", + .line_number = 0, + }; + DeepState_Begin(&test); + + return 0; +} + /* Notify that we're about to begin a test while running under Dr. Fuzz. */ void DeepState_BeginDrFuzz(struct DeepState_TestInfo *test) { DeepState_DrFuzzTest = test; From 281c5117edaf04c89f831822f74ad568b4233dad Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Mon, 19 Feb 2018 11:46:38 -0800 Subject: [PATCH 03/14] Add example for `DeepState_TakeOver()` --- examples/CMakeLists.txt | 3 +++ examples/TakeOver.cpp | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 examples/TakeOver.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 746b4ce..8db60a0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -43,3 +43,6 @@ target_link_libraries(StreamingAndFormatting deepstate) add_executable(Squares Squares.c) target_link_libraries(Squares deepstate) set_target_properties(Squares PROPERTIES COMPILE_DEFINITIONS "DEEPSTATE_TEST") + +add_executable(TakeOver TakeOver.cpp) +target_link_libraries(TakeOver deepstate) diff --git a/examples/TakeOver.cpp b/examples/TakeOver.cpp new file mode 100644 index 0000000..d31db80 --- /dev/null +++ b/examples/TakeOver.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018 Trail of Bits, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include + +using namespace deepstate; + +DEEPSTATE_NOINLINE void func(uint32_t x) { + CHECK_LT(x, 0x1234) + << "Found x=" << x << " was not greater than 0x1234."; + + if (x < 0x1234) { + printf("hi\n"); + } else { + printf("bye\n"); + } +} + +int main(int argc, char *argv[]) { + DeepState_InitOptions(argc, argv); + + uint32_t x = 123; + func(x); // Unexplored + + DeepState_TakeOver(); + + Symbolic y; + Symbolic z; + func(y); // Explored + func(z); // Explored +} From 76965704b42ad5fc2e96876ed2f43c38a5741f45 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Mon, 19 Feb 2018 12:09:38 -0800 Subject: [PATCH 04/14] Add `TakeOver` impl to angr executor - Add `--take_over` flag - Allow running tests from non-function instruction addr - Hook `DeepState_TakeOver()` with a `SimProcedure` that returns 1 - Make a fake test case that starts after `TakeOver` returns --- bin/deepstate/common.py | 4 ++ bin/deepstate/main_angr.py | 75 +++++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index b4e2966..db77def 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -121,6 +121,10 @@ class DeepState(object): "--output_test_dir", default="out", type=str, required=False, help="Directory where tests will be saved.") + parser.add_argument( + "--take_over", action='store_true', + help="Explore the program starting at the `TakeOver` hook.") + parser.add_argument( "binary", type=str, help="Path to the test binary to run.") diff --git a/bin/deepstate/main_angr.py b/bin/deepstate/main_angr.py index 28a651c..cade611 100644 --- a/bin/deepstate/main_angr.py +++ b/bin/deepstate/main_angr.py @@ -19,7 +19,7 @@ import logging import multiprocessing import sys import traceback -from .common import DeepState +from .common import DeepState, TestInfo L = logging.getLogger("deepstate.angr") L.setLevel(logging.INFO) @@ -261,12 +261,22 @@ class Log(angr.SimProcedure): DeepAngr(procedure=self).api_log(level, ea) -def do_run_test(project, test, apis, run_state): +class TakeOver(angr.SimProcedure): + def run(self): + """Do nothing, returning 1 to indicate that `DeepState_TakeOver()` has + been hooked for symbolic execution.""" + return 1 + + +def do_run_test(project, test, apis, run_state, should_call_state): """Symbolically executes a single test function.""" - test_state = project.factory.call_state( - test.ea, - base_state=run_state) + if should_call_state: + test_state = project.factory.call_state( + test.ea, + base_state=run_state) + else: + test_state = run_state mc = DeepAngr(state=test_state) mc.begin_test(test) @@ -291,10 +301,10 @@ def do_run_test(project, test, apis, run_state): da.crash_test() da.report() -def run_test(project, test, apis, run_state): +def run_test(project, test, apis, run_state, should_call_state=True): """Symbolically executes a single test function.""" try: - do_run_test(project, test, apis, run_state) + do_run_test(project, test, apis, run_state, should_call_state) except Exception as e: L.error("Uncaught exception: {}\n{}".format(e, traceback.format_exc())) @@ -350,6 +360,13 @@ def hook_apis(project, run_state): return mc, apis +def main_take_over(args, project): + takeover_ea = find_symbol_ea(project, 'DeepState_TakeOver') + + hook_function(project, takeover_ea, TakeOver) + + if not takeover_ea: + L.critical("Cannot find symbol `DeepState_TakeOver` in binary `{}`".format( args.binary)) return 1 @@ -359,16 +376,23 @@ def hook_apis(project, run_state): addr_size_bits = entry_state.arch.bits - # Concretely execute up until `DeepState_Setup`. + # Concretely execute up until `DeepState_TakeOver`. concrete_manager = angr.SimulationManager( project=project, active_states=[entry_state]) - concrete_manager.explore(find=setup_ea) + concrete_manager.explore(find=takeover_ea) try: - run_state = concrete_manager.found[0] + takeover_state = concrete_manager.found[0] except: - L.critical("Execution never hit `DeepState_Setup` in binary `{}`".format( + L.critical("Execution never hit `DeepState_TakeOver` in binary `{}`".format( + args.binary)) + return 1 + + try: + run_state = takeover_state.step().successors[0] + except: + L.critical("Unable to exit from `DeepState_TakeOver` in binary `{}`".format( args.binary)) return 1 @@ -381,28 +405,10 @@ def hook_apis(project, run_state): L.critical("Could not find API table in binary `{}`".format(args.binary)) return 1 - mc = DeepAngr(state=run_state) - apis = mc.read_api_table(ea_of_api_table) - - # Hook various functions. - hook_function(project, apis['IsSymbolicUInt'], IsSymbolicUInt) - hook_function(project, apis['ConcretizeData'], ConcretizeData) - hook_function(project, apis['ConcretizeCStr'], ConcretizeCStr) - hook_function(project, apis['MinUInt'], MinUInt) - hook_function(project, apis['MaxUInt'], MaxUInt) - hook_function(project, apis['Assume'], Assume) - hook_function(project, apis['Pass'], Pass) - hook_function(project, apis['Crash'], Crash) - hook_function(project, apis['Fail'], Fail) - hook_function(project, apis['Abandon'], Abandon) - hook_function(project, apis['SoftFail'], SoftFail) - hook_function(project, apis['Log'], Log) - hook_function(project, apis['StreamInt'], StreamInt) - hook_function(project, apis['StreamFloat'], StreamFloat) - hook_function(project, apis['StreamString'], StreamString) - hook_function(project, apis['ClearStream'], ClearStream) - hook_function(project, apis['LogStream'], LogStream) + _, apis = hook_apis(project, run_state) + fake_test = TestInfo(takeover_ea, '_takeover_test', '_takeover_file', 0) + return run_test(project, fake_test, apis, run_state, should_call_state=False) def main_unit_test(args, project): @@ -479,7 +485,10 @@ def main(): args.binary, e)) return 1 - return main_unit_test(args, project) + if args.take_over: + return main_take_over(args, project) + else: + return main_unit_test(args, project) if "__main__" == __name__: From 353bed10ee12c504faaa01e4fdca574f9d6f8380 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Mon, 19 Feb 2018 12:37:07 -0800 Subject: [PATCH 05/14] Factor out unit test running in Manticore executor --- bin/deepstate/main_manticore.py | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index dc7fe2f..f1d66d3 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -21,7 +21,7 @@ import sys try: import manticore except Exception as e: - if "Z3NotFoundError" in repr(type(e)): + if "Z3NotFoundError" in repr(type(e)): print "Manticore requires Z3 to be installed." sys.exit(255) else: @@ -364,22 +364,7 @@ def run_tests(args, state, apis): exit(0) -def main(): - args = DeepManticore.parse_args() - - try: - m = manticore.Manticore(args.binary) - except Exception as e: - L.critical("Cannot create Manticore instance on binary {}: {}".format( - args.binary, e)) - return 1 - - m.verbosity(1) - - # Hack to get around current broken _get_symbol_address - m._binary_type = 'not elf' - m._binary_obj = m._initial_state.platform.elf - +def main_unit_test(m, args): setup_ea = find_symbol_ea(m, 'DeepState_Setup') if not setup_ea: L.critical("Cannot find symbol `DeepState_Setup` in binary `{}`".format( @@ -401,5 +386,24 @@ def main(): m.run() +def main(): + args = DeepManticore.parse_args() + + try: + m = manticore.Manticore(args.binary) + except Exception as e: + L.critical("Cannot create Manticore instance on binary {}: {}".format( + args.binary, e)) + return 1 + + m.verbosity(1) + + # Hack to get around current broken _get_symbol_address + m._binary_type = 'not elf' + m._binary_obj = m._initial_state.platform.elf + + return main_unit_test(m, args) + + if "__main__" == __name__: exit(main()) From 9e7266399caabef0f7b75a167bb6cce92682c674 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Mon, 19 Feb 2018 18:04:06 -0800 Subject: [PATCH 06/14] Add `TakeOver` impl to Manticore executor --- bin/deepstate/main_manticore.py | 38 +++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index f1d66d3..cf66d24 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -28,7 +28,7 @@ except Exception as e: raise import multiprocessing import traceback -from .common import DeepState +from .common import DeepState, TestInfo from manticore.core.state import TerminateState from manticore.utils.helpers import issymbolic @@ -246,6 +246,12 @@ def hook_Log(state, level, ea): DeepManticore(state).api_log(level, ea) +def hook_TakeOver(state): + """Implements `DeepState_TakeOver`, returning 1 to indicate that it was + hooked for symbolic execution.""" + return 1 + + def hook(func): return lambda state: state.invoke_model(func) @@ -364,6 +370,30 @@ def run_tests(args, state, apis): exit(0) +def main_takeover(m, args): + takeover_ea = find_symbol_ea(m, 'DeepState_TakeOver') + if not takeover_ea: + L.critical("Cannot find symbol `DeepState_TakeOver` in binary `{}`".format( + args.binary)) + return 1 + + takeover_state = m._initial_state + + mc = DeepManticore(takeover_state) + + ea_of_api_table = find_symbol_ea(m, 'DeepState_API') + if not ea_of_api_table: + L.critical("Could not find API table in binary `{}`".format(args.binary)) + return 1 + + apis = mc.read_api_table(ea_of_api_table) + del mc + + fake_test = TestInfo(takeover_ea, '_takeover_test', '_takeover_file', 0) + m.add_hook(takeover_ea, lambda state: run_test(state, apis, fake_test)) + m.run() + + def main_unit_test(m, args): setup_ea = find_symbol_ea(m, 'DeepState_Setup') if not setup_ea: @@ -382,6 +412,7 @@ def main_unit_test(m, args): apis = mc.read_api_table(ea_of_api_table) del mc + m.add_hook(setup_ea, lambda state: run_tests(args, state, apis)) m.run() @@ -402,7 +433,10 @@ def main(): m._binary_type = 'not elf' m._binary_obj = m._initial_state.platform.elf - return main_unit_test(m, args) + if args.take_over: + return main_takeover(m, args) + else: + return main_unit_test(m, args) if "__main__" == __name__: From 2ea978999b2bbf7b3316fce93d39e9ab353194b3 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 12:31:14 -0800 Subject: [PATCH 07/14] Replace `DeepState_TakeOver()` with Manticore function model Now we have API parity with the angr executor, and return 1 from `DeepState_TakeOver()` when it is hooked for symbolic execution under the Manticore backend. --- bin/deepstate/main_manticore.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index cf66d24..73036a1 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -338,6 +338,10 @@ def do_run_test(state, apis, test): m.add_hook(apis['ClearStream'], hook(hook_ClearStream)) m.add_hook(apis['LogStream'], hook(hook_LogStream)) + # Here we hook `DeepState_TakeOver()`, even if running unit tests. + # In that case, we simply will never hit this hooked function model. + m.add_hook(test.ea, hook(hook_TakeOver)) + m.subscribe('will_terminate_state', done_test) m.run() From 4d5e390c157fc707e42265f6c71251fffbdacd7d Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 13:16:44 -0800 Subject: [PATCH 08/14] Factor out native runs of saved cases for a single test --- src/include/deepstate/DeepState.h | 84 +++++++++++++++++-------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 121327f..7e41e5f 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -502,8 +502,8 @@ DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { /* Run a single saved test case with input initialized from the file * `name` in directory `dir`. */ static enum DeepState_TestRunResult -DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, const char *dir, - const char *name) { +DeepState_RunSavedTestCase(struct DeepState_TestInfo *test, const char *dir, + const char *name) { size_t path_len = 2 + sizeof(char) * (strlen(dir) + strlen(name)); char *path = (char *) malloc(path_len); if (path == NULL) { @@ -532,6 +532,50 @@ DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, const char *dir, return result; } +/* Run a single test many times, initialized against each saved test case in + * `FLAGS_input_test_dir`. */ +static int DeepState_RunSavedCasesForTest(struct DeepState_TestInfo *test) { + int num_failed_tests = 0; + const char *test_file_name = basename((char *) test->file_name); + + size_t test_case_dir_len = 3 + strlen(FLAGS_input_test_dir) + + strlen(test_file_name) + strlen(test->test_name); + char *test_case_dir = (char *) malloc(test_case_dir_len); + if (test_case_dir == NULL) { + DeepState_Abandon("Error allocating memory"); + } + snprintf(test_case_dir, test_case_dir_len, "%s/%s/%s", + FLAGS_input_test_dir, test_file_name, test->test_name); + + struct dirent *dp; + DIR *dir_fd; + + dir_fd = opendir(test_case_dir); + if (dir_fd == NULL) { + DeepState_LogFormat(DeepState_LogInfo, + "Skipping test `%s`, no saved test cases", + test->test_name); + free(test_case_dir); + return 0; + } + + /* Read generated test cases and run a test for each file found. */ + while ((dp = readdir(dir_fd)) != NULL) { + if (IsTestCaseFile(dp->d_name)) { + enum DeepState_TestRunResult result = + DeepState_RunSavedTestCase(test, test_case_dir, dp->d_name); + + if (result != DeepState_TestRunPass) { + num_failed_tests++; + } + } + } + closedir(dir_fd); + free(test_case_dir); + + return num_failed_tests; +} + /* Run tests with saved input from `FLAGS_input_test_dir`. * * For each test unit and case, see if there are input files in the @@ -544,41 +588,7 @@ static int DeepState_RunSavedTestCases(void) { DeepState_Setup(); for (test = DeepState_FirstTest(); test != NULL; test = test->prev) { - const char *test_file_name = basename((char *) test->file_name); - - size_t test_case_dir_len = 3 + strlen(FLAGS_input_test_dir) - + strlen(test_file_name) + strlen(test->test_name); - char *test_case_dir = (char *) malloc(test_case_dir_len); - if (test_case_dir == NULL) { - DeepState_Abandon("Error allocating memory"); - } - snprintf(test_case_dir, test_case_dir_len, "%s/%s/%s", - FLAGS_input_test_dir, test_file_name, test->test_name); - - struct dirent *dp; - DIR *dir_fd; - - dir_fd = opendir(test_case_dir); - if (dir_fd == NULL) { - DeepState_LogFormat(DeepState_LogInfo, - "Skipping test `%s`, no saved test cases", - test->test_name); - continue; - } - - /* Read generated test cases and run a test for each file found. */ - while ((dp = readdir(dir_fd)) != NULL) { - if (IsTestCaseFile(dp->d_name)) { - enum DeepState_TestRunResult result = - DeepState_DoRunSavedTestCase(test, test_case_dir, dp->d_name); - - if (result != DeepState_TestRunPass) { - num_failed_tests++; - } - } - } - closedir(dir_fd); - free(test_case_dir); + num_failed_tests += DeepState_RunSavedCasesForTest(test); } DeepState_Teardown(); From 93a95b0e964681e249fb91b90727154045051290 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 13:17:28 -0800 Subject: [PATCH 09/14] Use native fake test and file name that matches executors --- src/lib/DeepState.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 05b4b79..1260f28 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -368,8 +368,8 @@ int DeepState_TakeOver(void) { struct DeepState_TestInfo test = { .prev = NULL, .test_func = NULL, - .test_name = "<__TAKE_OVER_TEST>", - .file_name = "<__TAKE_OVER_FILE>", + .test_name = "__takeover_test", + .file_name = "__takeover_file", .line_number = 0, }; DeepState_Begin(&test); From 661c600d2bcd13b40a78179d3ec6290b2004c585 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 15:17:12 -0800 Subject: [PATCH 10/14] Add impl of native takeover test case replay --- src/lib/DeepState.c | 68 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 1260f28..47317a8 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -364,6 +364,67 @@ void DrMemFuzzFunc(volatile uint8_t *buff, size_t size) { } } +void DeepState_RunSavedTakeOverCases(jmp_buf env, + struct DeepState_TestInfo *test) { + int num_failed_tests = 0; + const char *test_case_dir = FLAGS_input_test_dir; + + DIR *dir_fd = opendir(test_case_dir); + if (dir_fd == NULL) { + DeepState_LogFormat(DeepState_LogInfo, + "Skipping test `%s`, no saved test cases", + test->test_name); + return; + } + + struct dirent *dp; + + /* Read generated test cases and run a test for each file found. */ + while ((dp = readdir(dir_fd)) != NULL) { + if (IsTestCaseFile(dp->d_name)) { + pid_t case_pid = fork(); + if (!case_pid) { + size_t path_len = 2 + sizeof(char) * (strlen(test_case_dir) + + strlen(dp->d_name)); + char *path = (char *) malloc(path_len); + if (path == NULL) { + DeepState_Abandon("Error allocating memory"); + } + snprintf(path, path_len, "%s/%s", test_case_dir, dp->d_name); + InitializeInputFromFile(path); + free(path); + + longjmp(env, 1); + } + + int wstatus; + waitpid(case_pid, &wstatus, 0); + + /* If we exited normally, the status code tells us if the test passed. */ + if (WIFEXITED(wstatus)) { + uint8_t status = WEXITSTATUS(wstatus); + + if (status) { + DeepState_LogFormat(DeepState_LogError, + "Failed: TakeOver test with data from `%s`", + dp->d_name); + } else { + DeepState_LogFormat(DeepState_LogInfo, + "Passed: TakeOver test with data from `%s`", + dp->d_name); + } + } else { + /* If here, we exited abnormally but didn't catch it in the signal + * handler, and thus the test failed due to a crash. */ + DeepState_LogFormat(DeepState_LogError, + "Crashed: TakeOver test with data from `%s`", + dp->d_name); + } + } + } + closedir(dir_fd); +} + int DeepState_TakeOver(void) { struct DeepState_TestInfo test = { .prev = NULL, @@ -372,7 +433,12 @@ int DeepState_TakeOver(void) { .file_name = "__takeover_file", .line_number = 0, }; - DeepState_Begin(&test); + // DeepState_Begin(&test); + jmp_buf env; + if (!setjmp(env)) { + DeepState_RunSavedTakeOverCases(env, &test); + exit(0); + } return 0; } From 065c97c2c00cc860b2e7c697bf05d2e67ee3dcf8 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 15:29:12 -0800 Subject: [PATCH 11/14] Add `--take_over` flag to native binaries --- src/include/deepstate/DeepState.h | 1 + src/lib/DeepState.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 7e41e5f..03376c0 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -55,6 +55,7 @@ DEEPSTATE_BEGIN_EXTERN_C DECLARE_string(input_test_dir); DECLARE_string(output_test_dir); +DECLARE_string(take_over); enum { DeepState_InputSize = 8192 diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 47317a8..bd257ef 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -30,6 +30,7 @@ DEFINE_uint(num_workers, 1, DEFINE_string(input_test_dir, "", "Directory of saved tests to run."); DEFINE_string(output_test_dir, "", "Directory where tests will be saved."); +DEFINE_string(take_over, "", "Replay test cases in take-over mode."); /* Pointer to the last registers DeepState_TestInfo data structure */ struct DeepState_TestInfo *DeepState_LastTestInfo = NULL; From a81f816d89182e49aeba1a9ada4651d662f33871 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 15:29:39 -0800 Subject: [PATCH 12/14] If in take-over mode, exit on fatal error In take-over mode, the "test" is the entire binary being executed as a child process. So, we want to exit, rather than trying to `longjmp()` to `DeepState_ReturnToRun`, which was never initialized. --- src/lib/DeepState.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index bd257ef..1e92df9 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -65,7 +65,13 @@ void DeepState_Crash(void) { DEEPSTATE_NORETURN void DeepState_Fail(void) { DeepState_TestFailed = 1; - longjmp(DeepState_ReturnToRun, 1); + + if (FLAGS_take_over) { + // We want to communicate the failure to a parent process, so exit. + exit(DeepState_TestRunFail); + } else { + longjmp(DeepState_ReturnToRun, 1); + } } /* Mark this test as passing. */ From 23dbbbdc57953fc2558cf494e621b412a0f863d2 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 15:38:41 -0800 Subject: [PATCH 13/14] Log all test run result cases in native take-over Warning: this does not work correctly with tests that soft fail, e.g. via a `CHECK` assertion. This is because the soft failures only update the child's `DeepState_TestFailed` global variable, but do not exit. What we will soon do is share memory with the child process, and derive the "test result" from that shared memeory. --- src/lib/DeepState.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 1e92df9..cd52b60 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -411,13 +411,25 @@ void DeepState_RunSavedTakeOverCases(jmp_buf env, if (WIFEXITED(wstatus)) { uint8_t status = WEXITSTATUS(wstatus); - if (status) { + switch (status) { + case DeepState_TestRunPass: + DeepState_LogFormat(DeepState_LogInfo, + "Passed: TakeOver test with data from `%s`", + dp->d_name); + break; + case DeepState_TestRunFail: DeepState_LogFormat(DeepState_LogError, "Failed: TakeOver test with data from `%s`", dp->d_name); - } else { - DeepState_LogFormat(DeepState_LogInfo, - "Passed: TakeOver test with data from `%s`", + break; + case DeepState_TestRunAbandon: + DeepState_LogFormat(DeepState_LogError, + "Abandoned: TakeOver test with data from `%s`", + dp->d_name); + break; + default: + DeepState_LogFormat(DeepState_LogError, + "Unknown exit code from test with data from `%s`", dp->d_name); } } else { From c09feec114312584ecbf3c3369337628a6a104d9 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 20 Feb 2018 15:52:14 -0800 Subject: [PATCH 14/14] Call `DeepState_Begin()` for each forked take-over test case --- src/lib/DeepState.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index cd52b60..3361455 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -391,6 +391,8 @@ void DeepState_RunSavedTakeOverCases(jmp_buf env, if (IsTestCaseFile(dp->d_name)) { pid_t case_pid = fork(); if (!case_pid) { + DeepState_Begin(test); + size_t path_len = 2 + sizeof(char) * (strlen(test_case_dir) + strlen(dp->d_name)); char *path = (char *) malloc(path_len); @@ -452,7 +454,7 @@ int DeepState_TakeOver(void) { .file_name = "__takeover_file", .line_number = 0, }; - // DeepState_Begin(&test); + jmp_buf env; if (!setjmp(env)) { DeepState_RunSavedTakeOverCases(env, &test);