diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2d4e81d..28bc3e6 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -28,8 +28,8 @@ target_link_libraries(Euler deepstate) add_executable(IntegerOverflow IntegerOverflow.cpp) target_link_libraries(IntegerOverflow deepstate) -add_executable(IntegerArtihmetic IntegerArithmetic.cpp) -target_link_libraries(IntegerArtihmetic deepstate) +add_executable(IntegerArithmetic IntegerArithmetic.cpp) +target_link_libraries(IntegerArithmetic deepstate) add_executable(Lists Lists.cpp) target_link_libraries(Lists deepstate) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 33b3372..e60218a 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -18,12 +18,16 @@ #define SRC_INCLUDE_DEEPSTATE_DEEPSTATE_H_ #include +#include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -48,8 +52,20 @@ DEEPSTATE_BEGIN_EXTERN_C -DECLARE_string(output_test_dir); DECLARE_string(input_test_dir); +DECLARE_string(output_test_dir); + +enum { + DeepState_InputSize = 8192 +}; + +/* Byte buffer that will contain symbolic data that is used to supply requests + * for symbolic values (e.g. `int`s). */ +extern volatile uint8_t DeepState_Input[DeepState_InputSize]; + +/* Index into the `DeepState_Input` array that tracks how many input bytes have + * been consumed. */ +extern uint32_t DeepState_InputIndex; /* Return a symbolic value of a given type. */ extern int DeepState_Bool(void); @@ -332,11 +348,178 @@ extern void DeepState_SaveFailingTest(void); /* Jump buffer for returning to `DeepState_Run`. */ extern jmp_buf DeepState_ReturnToRun; +/* Checks a filename to see if might be a saved test case. + * + * Valid saved test cases have the suffix `.pass` or `.fail`. */ +static bool IsTestCaseFile(const char *name) { + const char *suffix = strchr(name, '.'); + if (suffix == NULL) { + return false; + } + + if (strcmp(suffix, ".pass") == 0 || strcmp(suffix, ".fail") == 0) { + return true; + } + + return false; +} + +/* Resets the global `DeepState_Input` buffer, then fills it with the + * data found in the file `path`. */ +static void InitializeInputFromFile(const char *path) { + struct stat stat_buf; + + FILE *fp = fopen(path, "r"); + if (fp == NULL) { + /* TODO(joe): Add error log with more info. */ + DeepState_Abandon("Unable to open file"); + } + + int fd = fileno(fp); + if (fd < 0) { + DeepState_Abandon("Tried to get file descriptor for invalid stream"); + } + if (fstat(fd, &stat_buf) < 0) { + DeepState_Abandon("Unable to access input file"); + }; + + if (stat_buf.st_size > sizeof(DeepState_Input)) { + /* TODO(joe): Add error log with more info. */ + DeepState_Abandon("File too large"); + } + + /* Reset the input buffer and reset the index. */ + memset((void *) DeepState_Input, 0, sizeof(DeepState_Input)); + DeepState_InputIndex = 0; + + size_t count = fread((void *) DeepState_Input, 1, stat_buf.st_size, fp); + fclose(fp); + + if (count != stat_buf.st_size) { + /* TODO(joe): Add error log with more info. */ + DeepState_Abandon("Error reading file"); + } + + DeepState_LogFormat(DeepState_LogInfo, + "Initialized test input buffer with data from `%s`", + path); +} + +/* Run a single saved test case with input initialized from the file + * `name` in directory `dir`. + * + * This function does not attempt to save test outcomes. */ +static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, + const char *dir, const char *name) { + int num_failed_tests = 0; + + size_t path_len = 2 + sizeof(char) * (strlen(dir) + strlen(name)); + char *path = (char *) malloc(path_len); + if (path == NULL) { + DeepState_Abandon("Error allocating memory"); + } + snprintf(path, path_len, "%s/%s", dir, name); + + InitializeInputFromFile(path); + + free(path); + + DeepState_Begin(test); + + /* Run the test. */ + if (!setjmp(DeepState_ReturnToRun)) { + /* Convert uncaught C++ exceptions into a test failure. */ +#if defined(__cplusplus) && defined(__cpp_exceptions) + try { +#endif /* __cplusplus */ + + test->test_func(); /* Run the test function. */ + DeepState_Pass(); + +#if defined(__cplusplus) && defined(__cpp_exceptions) + } catch(...) { + DeepState_Fail(); + } +#endif /* __cplusplus */ + + /* We caught a failure when running the test. */ + } else if (DeepState_CatchFail()) { + num_failed_tests = 1; + DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name); + + /* The test was abandoned. We may have gotten soft failures before + * abandoning, so we prefer to catch those first. */ + } else if (DeepState_CatchAbandoned()) { + DeepState_LogFormat(DeepState_LogFatal, "Abandoned: %s", test->test_name); + + /* The test passed. */ + } else { + DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); + } + + return num_failed_tests; +} + +/* Run tests with saved input from `FLAGS_input_test_dir`. + * + * For each test unit and case, see if there are input files in the + * expected directories. If so, use them to initialize + * `DeepState_Input`, then run the test. If not, skip the test. */ +static int DeepState_RunSavedTestCases(void) { + int num_failed_tests = 0; + struct DeepState_TestInfo *test = NULL; + + DeepState_Setup(); + + for (test = DeepState_FirstTest(); test != NULL; test = test->prev) { + const char *test_file_name = basename((char *) test->file_name); + + size_t test_case_dir_len = 3 + strlen(FLAGS_input_test_dir) + + strlen(test_file_name) + strlen(test->test_name); + char *test_case_dir = (char *) malloc(test_case_dir_len); + if (test_case_dir == NULL) { + DeepState_Abandon("Error allocating memory"); + } + snprintf(test_case_dir, test_case_dir_len, "%s/%s/%s", + FLAGS_input_test_dir, test_file_name, test->test_name); + + struct dirent *dp; + DIR *dir_fd; + + dir_fd = opendir(test_case_dir); + if (dir_fd == NULL) { + DeepState_LogFormat(DeepState_LogInfo, + "Skipping test `%s`, no saved test cases", + test->test_name); + continue; + } + + /* Read generated test cases and run a test for each file found. */ + while ((dp = readdir(dir_fd)) != NULL) { + if (IsTestCaseFile(dp->d_name)) { + num_failed_tests += DeepState_DoRunSavedTestCase(test, test_case_dir, + dp->d_name); + } + } + closedir(dir_fd); + free(test_case_dir); + } + + DeepState_Teardown(); + + return num_failed_tests; +} + /* Start DeepState and run the tests. Returns the number of failed tests. */ static int DeepState_Run(void) { if (!DeepState_OptionsAreInitialized) { DeepState_Abandon("Please call DeepState_InitOptions(argc, argv) in main."); } + + if (HAS_FLAG_input_test_dir) { + return DeepState_RunSavedTestCases(); + } + int num_failed_tests = 0; int use_drfuzz = getenv("DYNAMORIO_EXE_PATH") != NULL; struct DeepState_TestInfo *test = NULL; @@ -355,14 +538,13 @@ static int DeepState_Run(void) { } /* Run the test. */ if (!setjmp(DeepState_ReturnToRun)) { - /* Convert uncaught C++ exceptions into a test failure. */ #if defined(__cplusplus) && defined(__cpp_exceptions) try { #endif /* __cplusplus */ - test->test_func(); /* Run the test function. */ - DeepState_Pass(); + test->test_func(); /* Run the test function. */ + DeepState_Pass(); #if defined(__cplusplus) && defined(__cpp_exceptions) } catch(...) { diff --git a/src/include/deepstate/DeepState.hpp b/src/include/deepstate/DeepState.hpp index c41424c..208fecc 100644 --- a/src/include/deepstate/DeepState.hpp +++ b/src/include/deepstate/DeepState.hpp @@ -184,7 +184,7 @@ class Symbolic : public SymbolicLinearContainer { }; template -class Symbolic> : +class Symbolic> : public SymbolicLinearContainer> {}; #define MAKE_SYMBOL_SPECIALIZATION(Tname, tname) \ @@ -299,7 +299,7 @@ static T Pump(T val, unsigned max=10) { return val; } if (!max) { - DeepState_Abandon("Must have a positie maximum number of values to pump."); + DeepState_Abandon("Must have a positive maximum number of values to pump."); } for (auto i = 0U; i < max - 1; ++i) { T min_val = Minimize(val); diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 160493c..ac61b90 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -28,7 +28,7 @@ DEEPSTATE_BEGIN_EXTERN_C DEFINE_uint(num_workers, 1, "Number of workers to spawn for testing and test generation."); -DEFINE_string(input_test_dir, "", "Directory where tests will be saved."); +DEFINE_string(input_test_dir, "", "Directory of saved tests to run."); DEFINE_string(output_test_dir, "", "Directory where tests will be saved."); /* Pointer to the last registers DeepState_TestInfo data structure */ @@ -37,17 +37,9 @@ struct DeepState_TestInfo *DeepState_LastTestInfo = NULL; /* Pointer to the test being run in this process by Dr. Fuzz. */ static struct DeepState_TestInfo *DeepState_DrFuzzTest = NULL; -enum { - DeepState_InputSize = 8192 -}; - -/* Byte buffer that will contain symbolic data that is used to supply requests - * for symbolic values (e.g. `int`s). */ -static volatile uint8_t DeepState_Input[DeepState_InputSize]; - -/* Index into the `DeepState_Input` array that tracks how many input bytes have - * been consumed. */ -static uint32_t DeepState_InputIndex = 0; +/* Initialize global input buffer and index. */ +volatile uint8_t DeepState_Input[DeepState_InputSize] = {}; +uint32_t DeepState_InputIndex = 0; /* Jump buffer for returning to `DeepState_Run`. */ jmp_buf DeepState_ReturnToRun = {}; diff --git a/src/lib/Log.c b/src/lib/Log.c index 3b6fbe7..c027ac6 100644 --- a/src/lib/Log.c +++ b/src/lib/Log.c @@ -77,6 +77,9 @@ void DeepState_Log(enum DeepState_LogLevel level, const char *str) { if (DeepState_LogError == level) { DeepState_SoftFail(); } else if (DeepState_LogFatal == level) { + /* `DeepState_Fail()` calls `longjmp()`, so we need to make sure + * we clean up the log buffer first. */ + DeepState_ClearStream(level); DeepState_Fail(); } }