From 8ff5f6cac68191e8b4f7e47762df037eda885e5c Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 11:48:30 -0800 Subject: [PATCH 1/6] Fork when running saved test cases This anticipates support for crashing tests. --- src/include/deepstate/DeepState.h | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index e60218a..e3ad066 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -405,6 +406,45 @@ static void InitializeInputFromFile(const char *path) { path); } +/* Run a test case, assuming we have forked from the test harness to do so. + * + * An exit code of 0 indicates that the test passed. Any other exit + * code, or termination by a signal, indicates a test failure. */ +static void DeepState_ForkTest(struct DeepState_TestInfo *test) { + /* Run the test. */ + if (!setjmp(DeepState_ReturnToRun)) { + /* Convert uncaught C++ exceptions into a test failure. */ +#if defined(__cplusplus) && defined(__cpp_exceptions) + try { +#endif /* __cplusplus */ + + test->test_func(); /* Run the test function. */ + exit(0); + +#if defined(__cplusplus) && defined(__cpp_exceptions) + } catch(...) { + exit(1); + } +#endif /* __cplusplus */ + + /* We caught a failure when running the test. */ + } else if (DeepState_CatchFail()) { + DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name); + exit(1); + + /* The test was abandoned. We may have gotten soft failures before + * abandoning, so we prefer to catch those first. */ + } else if (DeepState_CatchAbandoned()) { + DeepState_LogFormat(DeepState_LogFatal, "Abandoned: %s", test->test_name); + exit(1); + + /* The test passed. */ + } else { + DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); + exit(0); + } +} + /* Run a single saved test case with input initialized from the file * `name` in directory `dir`. * @@ -426,35 +466,20 @@ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, DeepState_Begin(test); - /* Run the test. */ - if (!setjmp(DeepState_ReturnToRun)) { - /* Convert uncaught C++ exceptions into a test failure. */ -#if defined(__cplusplus) && defined(__cpp_exceptions) - try { -#endif /* __cplusplus */ + pid_t test_pid = fork(); + if (!test_pid) { + DeepState_ForkTest(test); + } + int wstatus; + waitpid(test_pid, &wstatus, 0); - test->test_func(); /* Run the test function. */ - DeepState_Pass(); - -#if defined(__cplusplus) && defined(__cpp_exceptions) - } catch(...) { - DeepState_Fail(); + if (WIFEXITED(wstatus)) { + uint8_t status = WEXITSTATUS(wstatus); + if (!status) { + num_failed_tests++; } -#endif /* __cplusplus */ - - /* We caught a failure when running the test. */ - } else if (DeepState_CatchFail()) { - num_failed_tests = 1; - DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name); - - /* The test was abandoned. We may have gotten soft failures before - * abandoning, so we prefer to catch those first. */ - } else if (DeepState_CatchAbandoned()) { - DeepState_LogFormat(DeepState_LogFatal, "Abandoned: %s", test->test_name); - - /* The test passed. */ } else { - DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); + num_failed_tests++; } return num_failed_tests; From 8f39961b3f8674b833055e03229e13ef498d54d4 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 12:18:36 -0800 Subject: [PATCH 2/6] Save result of forked test run if flag set --- src/include/deepstate/DeepState.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index e3ad066..4f9ab57 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -430,6 +430,9 @@ static void DeepState_ForkTest(struct DeepState_TestInfo *test) { /* We caught a failure when running the test. */ } else if (DeepState_CatchFail()) { DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name); + if (HAS_FLAG_output_test_dir) { + DeepState_SaveFailingTest(); + } exit(1); /* The test was abandoned. We may have gotten soft failures before @@ -441,14 +444,15 @@ static void DeepState_ForkTest(struct DeepState_TestInfo *test) { /* The test passed. */ } else { DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); + if (HAS_FLAG_output_test_dir) { + DeepState_SavePassingTest(); + } exit(0); } } /* Run a single saved test case with input initialized from the file - * `name` in directory `dir`. - * - * This function does not attempt to save test outcomes. */ + * `name` in directory `dir`. */ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, const char *dir, const char *name) { int num_failed_tests = 0; From b61b3e7f2d05e65b91cd77c65568926ba633220a Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 12:19:09 -0800 Subject: [PATCH 3/6] Fork for all test runs --- src/include/deepstate/DeepState.h | 43 ++++++++----------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 4f9ab57..5b4d1b1 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -565,42 +565,21 @@ static int DeepState_Run(void) { } else { DeepState_Begin(test); } - /* Run the test. */ - if (!setjmp(DeepState_ReturnToRun)) { - /* Convert uncaught C++ exceptions into a test failure. */ -#if defined(__cplusplus) && defined(__cpp_exceptions) - try { -#endif /* __cplusplus */ - test->test_func(); /* Run the test function. */ - DeepState_Pass(); + pid_t test_pid = fork(); + if (!test_pid) { + DeepState_ForkTest(test); + } + int wstatus; + waitpid(test_pid, &wstatus, 0); -#if defined(__cplusplus) && defined(__cpp_exceptions) - } catch(...) { - DeepState_Fail(); + if (WIFEXITED(wstatus)) { + uint8_t status = WEXITSTATUS(wstatus); + if (!status) { + num_failed_tests++; } -#endif /* __cplusplus */ - - - /* We caught a failure when running the test. */ - } else if (DeepState_CatchFail()) { - ++num_failed_tests; - DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name); - if (HAS_FLAG_output_test_dir) { - DeepState_SaveFailingTest(); - } - - /* The test was abandoned. We may have gotten soft failures before - * abandoning, so we prefer to catch those first. */ - } else if (DeepState_CatchAbandoned()) { - DeepState_LogFormat(DeepState_LogFatal, "Abandoned: %s", test->test_name); - - /* The test passed. */ } else { - DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); - if (HAS_FLAG_output_test_dir) { - DeepState_SavePassingTest(); - } + num_failed_tests++; } } From 3c5d5daeaf9a82256c00749518915a1285a65f11 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 14:11:55 -0800 Subject: [PATCH 4/6] Use more accurate function name --- src/include/deepstate/DeepState.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 5b4d1b1..cd8de42 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -410,7 +410,7 @@ static void InitializeInputFromFile(const char *path) { * * An exit code of 0 indicates that the test passed. Any other exit * code, or termination by a signal, indicates a test failure. */ -static void DeepState_ForkTest(struct DeepState_TestInfo *test) { +static void DeepState_RunTest(struct DeepState_TestInfo *test) { /* Run the test. */ if (!setjmp(DeepState_ReturnToRun)) { /* Convert uncaught C++ exceptions into a test failure. */ @@ -472,7 +472,7 @@ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, pid_t test_pid = fork(); if (!test_pid) { - DeepState_ForkTest(test); + DeepState_RunTest(test); } int wstatus; waitpid(test_pid, &wstatus, 0); @@ -568,7 +568,7 @@ static int DeepState_Run(void) { pid_t test_pid = fork(); if (!test_pid) { - DeepState_ForkTest(test); + DeepState_RunTest(test); } int wstatus; waitpid(test_pid, &wstatus, 0); From a68520642a95d5ab710229b2e9e06eef91786f09 Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Tue, 13 Feb 2018 14:22:59 -0800 Subject: [PATCH 5/6] Fully factor out forking and running a single test --- src/include/deepstate/DeepState.h | 58 +++++++++++++------------------ 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index cd8de42..dff969e 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -451,12 +451,32 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { } } +/* Fork and run `test`. + * + * Returns 1 if the test failed, 0 otherwise. */ +static int DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { + pid_t test_pid = fork(); + if (!test_pid) { + DeepState_RunTest(test); + } + int wstatus; + waitpid(test_pid, &wstatus, 0); + + /* If we exited normally, the status code tells us if the test passed. */ + if (WIFEXITED(wstatus)) { + uint8_t status = WEXITSTATUS(wstatus); + + return status ? 1 : 0; + } + + /* If here, we exited abnormally, and so the test failed. */ + return 1; +} + /* Run a single saved test case with input initialized from the file * `name` in directory `dir`. */ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, const char *dir, const char *name) { - int num_failed_tests = 0; - size_t path_len = 2 + sizeof(char) * (strlen(dir) + strlen(name)); char *path = (char *) malloc(path_len); if (path == NULL) { @@ -470,23 +490,7 @@ static int DeepState_DoRunSavedTestCase(struct DeepState_TestInfo *test, DeepState_Begin(test); - pid_t test_pid = fork(); - if (!test_pid) { - DeepState_RunTest(test); - } - int wstatus; - waitpid(test_pid, &wstatus, 0); - - if (WIFEXITED(wstatus)) { - uint8_t status = WEXITSTATUS(wstatus); - if (!status) { - num_failed_tests++; - } - } else { - num_failed_tests++; - } - - return num_failed_tests; + return DeepState_ForkAndRunTest(test); } /* Run tests with saved input from `FLAGS_input_test_dir`. @@ -566,21 +570,7 @@ static int DeepState_Run(void) { DeepState_Begin(test); } - pid_t test_pid = fork(); - if (!test_pid) { - DeepState_RunTest(test); - } - int wstatus; - waitpid(test_pid, &wstatus, 0); - - if (WIFEXITED(wstatus)) { - uint8_t status = WEXITSTATUS(wstatus); - if (!status) { - num_failed_tests++; - } - } else { - num_failed_tests++; - } + num_failed_tests += DeepState_ForkAndRunTest(test); } if (use_drfuzz) { From 68595a949364af4faac7b4aa6855d11b015c752e Mon Sep 17 00:00:00 2001 From: Joe Ranweiler Date: Wed, 14 Feb 2018 10:10:51 -0800 Subject: [PATCH 6/6] Fix handling of test failures due to uncaught exceptions We don't just want to exit here, but `longjmp()` back to the conditional via a call to `DeepState_Fail()`. In doing so, we end up in the common `DeepState_CatchFail()` branch, which exits with the same nonzero error code, but also saves the test case if appropriate. --- src/include/deepstate/DeepState.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index dff969e..7194376 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -423,7 +423,7 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { #if defined(__cplusplus) && defined(__cpp_exceptions) } catch(...) { - exit(1); + DeepState_Fail(); } #endif /* __cplusplus */