diff --git a/README.md b/README.md index 3b78b9f..c452288 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,9 @@ The [2018 IEEE Cybersecurity Development Conference](https://secdev.ieee.org/201 ## Supported Platforms -DeepState currently targets Linux, with macOS support in progress. +DeepState currently targets Linux, with macOS support in progress +(some fuzzers work fine, but symbolic execution is not well-supported +yet, without a painful cross-compilation process). ## Dependencies @@ -113,6 +115,26 @@ argument to see all DeepState options. DeepState consists of a static library, used to write test harnesses, and command-line _executors_ written in Python. At this time, the best documentation is in the [examples](/examples) and in our [paper](https://agroce.github.io/bar18.pdf). A more extensive example, using DeepState and libFuzzer to test a user-mode file system, is available [here](https://github.com/agroce/testfs); in particular the [Tests.cpp](https://github.com/agroce/testfs/blob/master/Tests.cpp) file and CMakeLists.txt show DeepState usage. +## Built-In Fuzzer + +Every DeepState executable provides a simple built-in fuzzer that +generates tests using completely random data. Using this fuzzer is as +simple as calling the native executable with the `--fuzz` argument. +The fuzzer also takes a `seed` and `timeout` (default of two minutes) +to control the fuzzing. If you want to actually save the test cases +generated, you need to add a `--output_test_dir` arument to tell +DeepState where to put the generated tests. By default fuzzing saves +only failing and crashing tests. One more command line argument that +is particularly useful for fuzzing is the `--log_level` argument, +which controls how much output each test produces. By default, this +is set to show all the logging from your tests, which slows fuzzing, +but setting it to 2 or higher will only show messages produced by +tests failing or crashing. + +Note that while symbolic execution only works on Linux, without a +fairly complex cross-compliation process, the brute force fuzzer works +on macOS or (as far as we know) any Unix-like system. + ## Fuzzing with libFuzzer If you install clang 6.0 or later, and run `cmake` when you install @@ -121,7 +143,7 @@ generate tests using libFuzzer. Because both DeepState and libFuzzer want to be `main`, this requires building a different executable for libFuzzer. The `examples` directory shows how this can be done. The libFuzzer executable works like any other libFuzzer executable, and -the tests produced can be run using the normal DeepState executable. +the tests produced can be replayed using the normal DeepState executable. For example, generating some tests of the `OneOf` example (up to 5,000 runs), then running those tests to examine the results, would look like: @@ -143,7 +165,10 @@ corpus, but fuzzing will work even without an initial corpus, unlike AFL. One hint when using libFuzzer is to avoid dynamically allocating memory during a test, if that memory would not be freed on a test failure. This will leak memory and libFuzzer will run out of memory -very quickly in each fuzzing session. +very quickly in each fuzzing session. In theory, libFuzzer will work +on macOS, but getting everything to build with the right version of +clang can be difficult, since the Apple-provided LLVM is unlikely to +support libFuzzer on many versions of the operating system. ## Test case reduction @@ -196,6 +221,8 @@ WRITING REDUCED TEST WITH 20 BYTES TO minrmdirfail.test You can use `--which_test ` to specify which test to run, as with the `--input_which_test` options to test replay. +Test case reduction should work on any OS. + ## Fuzzing with AFL DeepState can also be used with a file-based fuzzer (e.g. AFL). There @@ -249,6 +276,10 @@ deferred instrumentation. You'll need code like: just before the call to `DeepState_Run()` (which reads the entire input file) in your `main`. +Because AFL and other file-based fuzzers only rely on the DeepState +native test executable, they should (like DeepState's built-in simple +fuzzer) work fine on macOS and other Unix-like OSes. + ## Contributing All accepted PRs are awarded bounties by Trail of Bits. Join the #deepstate channel on the [Empire Hacking Slack](https://empireslacking.herokuapp.com/) to discuss ongoing development and claim bounties. Check the [good first issue](https://github.com/trailofbits/deepstate/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) label for suggested contributions. diff --git a/bin/deepstate/reducer.py b/bin/deepstate/reducer.py index fae6299..87aa4ae 100644 --- a/bin/deepstate/reducer.py +++ b/bin/deepstate/reducer.py @@ -130,6 +130,8 @@ def main(): cuts = s[0] for c in cuts: newTest = currentTest[:c[0]] + currentTest[c[1]+1:] + if len(newTest) == len(currentTest): + continue # Ignore non-shrinking reductions r = writeAndRunCandidate(newTest) if checks(r): print("ONEOF REMOVAL REDUCED TEST TO", len(newTest), "BYTES") diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index c103a72..2059824 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -63,8 +64,12 @@ DECLARE_string(output_test_dir); DECLARE_bool(take_over); DECLARE_bool(abort_on_fail); DECLARE_bool(verbose_reads); +DECLARE_bool(fuzz); +DECLARE_bool(fuzz_save_passing); DECLARE_int(log_level); +DECLARE_int(seed); +DECLARE_int(timeout); enum { DeepState_InputSize = 8192 @@ -519,7 +524,9 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { } else { DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); if (HAS_FLAG_output_test_dir) { - DeepState_SavePassingTest(); + if (!FLAGS_fuzz || FLAGS_fuzz_save_passing) { + DeepState_SavePassingTest(); + } } exit(DeepState_TestRunPass); } @@ -588,6 +595,37 @@ DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { return DeepState_TestRunCrash; } +/* Run a test case with input initialized by fuzzing. */ +static enum DeepState_TestRunResult +DeepState_FuzzOneTestCase(struct DeepState_TestInfo *test) { + DeepState_InputIndex = 0; + + for (int i = 0; i < DeepState_InputSize; i++) { + DeepState_Input[i] = (char)rand(); + } + + DeepState_Begin(test); + + enum DeepState_TestRunResult result = DeepState_ForkAndRunTest(test); + + if (result == DeepState_TestRunCrash) { + DeepState_LogFormat(DeepState_LogError, "Crashed: %s", test->test_name); + + if (HAS_FLAG_output_test_dir) { + DeepState_SaveCrashingTest(); + } + + DeepState_Crash(); + } + + if (FLAGS_abort_on_fail && ((result == DeepState_TestRunCrash) || + (result == DeepState_TestRunFail))) { + abort(); + } + + return result; +} + /* Run a single saved test case with input initialized from the file * `name` in directory `dir`. */ static enum DeepState_TestRunResult @@ -711,6 +749,60 @@ static int DeepState_RunSingleSavedTestCase(void) { return num_failed_tests; } +/* Fuzz test `FLAGS_input_which_test` or first test, if not defined. */ +static int DeepState_Fuzz(void) { + DeepState_LogFormat(DeepState_LogInfo, "Starting fuzzing"); + + if (HAS_FLAG_seed) { + srand(FLAGS_seed); + } else { + srand(time(NULL)); + } + + long start = (long)time(NULL); + long current = (long)time(NULL); + long diff = 0; + unsigned i = 0; + + int num_failed_tests = 0; + + struct DeepState_TestInfo *test = NULL; + + DeepState_Setup(); + + for (test = DeepState_FirstTest(); test != NULL; test = test->prev) { + if (HAS_FLAG_input_which_test) { + if (strncmp(FLAGS_input_which_test, test->test_name, strlen(FLAGS_input_which_test)) == 0) { + break; + } + } else { + DeepState_LogFormat(DeepState_LogInfo, + "No test specified, defaulting to last test defined"); + break; + } + } + + if (test == NULL) { + DeepState_LogFormat(DeepState_LogInfo, + "Could not find matching test for %s", + FLAGS_input_which_test); + return 0; + } + + while (diff < FLAGS_timeout) { + i++; + num_failed_tests += DeepState_FuzzOneTestCase(test); + + current = (long)time(NULL); + diff = current-start; + } + + DeepState_LogFormat(DeepState_LogInfo, "Ran %u tests. %d failed tests.", + i, num_failed_tests); + + return num_failed_tests; +} + /* Run tests from `FLAGS_input_test_files_dir`, under `FLAGS_input_which_test` * or first test, if not defined. */ static int DeepState_RunSingleSavedTestDir(void) { @@ -811,7 +903,11 @@ static int DeepState_Run(void) { if (HAS_FLAG_input_test_files_dir) { return DeepState_RunSingleSavedTestDir(); - } + } + + if (FLAGS_fuzz) { + return DeepState_Fuzz(); + } int num_failed_tests = 0; int use_drfuzz = getenv("DYNAMORIO_EXE_PATH") != NULL; diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index fdf40ad..267386c 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -37,8 +37,12 @@ DEFINE_string(output_test_dir, "", "Directory where tests will be saved."); DEFINE_bool(take_over, false, "Replay test cases in take-over mode."); DEFINE_bool(abort_on_fail, false, "Abort on file replay failure (useful in file fuzzing)."); DEFINE_bool(verbose_reads, false, "Report on bytes being read during execution of test."); +DEFINE_bool(fuzz, false, "Perform brute force unguided fuzzing."); +DEFINE_bool(fuzz_save_passing, false, "Save passing tests during fuzzing."); DEFINE_int(log_level, 0, "Minimum level of logging to output."); +DEFINE_int(seed, 0, "Seed for brute force fuzzing (uses time if not set)."); +DEFINE_int(timeout, 120, "Timeout for brute force fuzzing."); /* Set to 1 by Manticore/Angr/etc. when we're running symbolically. */ int DeepState_UsingSymExec = 0; @@ -555,14 +559,60 @@ void DeepState_BeginDrFuzz(struct DeepState_TestInfo *test) { DrMemFuzzFunc(DeepState_Input, DeepState_InputSize); } +/* Right now "fake" a hexdigest by just using random bytes. Not ideal. */ +void makeFilename(char *name, size_t size) { + const char *entities = "0123456789abcdef"; + for (int i = 0; i < size; i++) { + name[i] = entities[rand()%16]; + } +} + +void writeInputData(char* name) { + size_t path_len = 2 + sizeof(char) * (strlen(FLAGS_output_test_dir) + strlen(name)); + char *path = (char *) malloc(path_len); + snprintf(path, path_len, "%s/%s", FLAGS_output_test_dir, name); + FILE *fp = fopen(path, "wb"); + if (fp == NULL) { + DeepState_LogFormat(DeepState_LogError, "Failed to create file `%s`", path); + free(path); + return; + } + size_t written = fwrite((void *)DeepState_Input, 1, DeepState_InputSize, fp); + if (written != DeepState_InputSize) { + DeepState_LogFormat(DeepState_LogError, "Failed to write to file `%s`", path); + } else { + DeepState_LogFormat(DeepState_LogInfo, "Saved test case to file `%s`", path); + } + free(path); + fclose(fp); +} + /* Save a passing test to the output test directory. */ -void DeepState_SavePassingTest(void) {} +void DeepState_SavePassingTest(void) { + char name[48]; + makeFilename(name, 40); + name[40] = 0; + strncat(name, ".pass", 48); + writeInputData(name); +} /* Save a failing test to the output test directory. */ -void DeepState_SaveFailingTest(void) {} +void DeepState_SaveFailingTest(void) { + char name[48]; + makeFilename(name, 40); + name[40] = 0; + strncat(name, ".fail", 48); + writeInputData(name); +} /* Save a crashing test to the output test directory. */ -void DeepState_SaveCrashingTest(void) {} +void DeepState_SaveCrashingTest(void) { + char name[48]; + makeFilename(name, 40); + name[40] = 0; + strncat(name, ".crash", 48); + writeInputData(name); +} /* Return the first test case to run. */ struct DeepState_TestInfo *DeepState_FirstTest(void) { diff --git a/src/lib/Option.c b/src/lib/Option.c index 9401006..5a0c8ed 100644 --- a/src/lib/Option.c +++ b/src/lib/Option.c @@ -125,7 +125,8 @@ static void ProcessOptionString(void) { if (' ' == *ch || DeepState_FakeSpace == *ch) { *ch = '\0'; state = kSeenSpace; - + } else if ('-' == *ch) { + state = kSeenDash; } else if (IsValidValueChar(*ch)) { /* E.g. `--tools bbcount`. */ state = kInValue; DeepState_OptionValues[num_options - 1] = ch;