diff --git a/CMakeLists.txt b/CMakeLists.txt index 64cfe8a..de68a02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,12 @@ cmake_minimum_required(VERSION 2.8) enable_language(C) enable_language(CXX) +set(BUILD_LIBFUZZER "$ENV{BUILD_LIBFUZZER}") +if (BUILD_LIBFUZZER) + SET(CMAKE_C_COMPILER clang) + SET(CMAKE_CXX_COMPILER clang++) +endif() + set(CMAKE_POSITION_INDEPENDENT_CODE ON) if(NOT CMAKE_BUILD_TYPE) @@ -39,7 +45,7 @@ set(CMAKE_CXX_EXTENSIONS ON) if (NOT WIN32) set(CMAKE_C_STANDARD 99) set(CMAKE_CXX_STANDARD 11) -endif () +endif() add_library(${PROJECT_NAME} STATIC src/lib/DeepState.c @@ -71,7 +77,7 @@ target_include_directories(${PROJECT_NAME}32 PUBLIC SYSTEM "${CMAKE_SOURCE_DIR}/src/include" ) -# Install the +# Install the library install( DIRECTORY "${CMAKE_SOURCE_DIR}/src/include/deepstate" DESTINATION include @@ -84,6 +90,28 @@ install( ARCHIVE DESTINATION lib ) +if (BUILD_LIBFUZZER) + add_library(${PROJECT_NAME}_LF STATIC + src/lib/DeepState.c + src/lib/Log.c + src/lib/Option.c + src/lib/Stream.c + ) + + target_compile_options(${PROJECT_NAME}_LF PUBLIC -mno-avx -fsanitize=fuzzer-no-link) + + target_include_directories(${PROJECT_NAME}_LF + PUBLIC SYSTEM "${CMAKE_SOURCE_DIR}/src/include" + ) + + install( + TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_LF + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + ) +endif() + + set(SETUP_PY_IN "${CMAKE_SOURCE_DIR}/bin/setup.py.in") set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py") configure_file(${SETUP_PY_IN} ${SETUP_PY}) diff --git a/README.md b/README.md index 7afe4f3..7ffaa43 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The [2018 IEEE Cybersecurity Development Conference](https://secdev.ieee.org/201 * Tests look like Google Test, but can use symbolic execution/fuzzing to generate data (parameterized unit testing) * Easier to learn than binary analysis tools/fuzzers, but provides similar functionality -* Already supports Manticore, Angr, Dr. Fuzz, file-based fuzzing with e.g., AFL; more back-ends likely in future +* Already supports Manticore, Angr, libFuzzer, file-based fuzzing with e.g., AFL; more back-ends likely in future * Switch test generation tool without re-writing test harness * Work around show-stopper bugs * Find out which tool works best for your code under test @@ -89,9 +89,41 @@ deepstate-angr --num_workers 4 --output_test_dir out $DEEPSTATE/build/examples/I DeepState consists of a static library, used to write test harnesses, and command-line _executors_ written in Python. At this time, the best documentation is in the [examples](/examples) and in our [paper](https://agroce.github.io/bar18.pdf). -## Fuzzing +## Fuzzing with libFuzzer -DeepState now can be used with a file-based fuzzer (e.g. AFL). There +If you install clang 6.0 or later, and run `cmake` when you install +with the `BUILD_LIBFUZZER` environment variable defined, you can +generate tests using LlibFuzzer. Because both DeepState and libFuzzer +want to be `main`, this requires building a different executable for +libFuzzer. The `examples` directory shows how this can be done. The +libFuzzer executable works like any other libFuzzer executable, and +the tests produced can be run using the normal DeepState executable. +For example, generating some tests of the `OneOf` example (up to 5,000 +runs), then running those tests to examine the results, would look +like: + +```shell +mkdir OneOf_libFuzzer_corpus +./OneOf_LF -runs=5000 OneOf_libFuzzer_corpus +./OneOf --input_test_files_dir OneOf_libFuzzer_corpus +``` + +Use the `LIBFUZZER_WHICH_TEST` +environment variable to control which test libFuzzer runs, using a +fully qualified name (e.g., +`Arithmetic_InvertibleMultiplication_CanFail`). By default, you get +the last test defined (which works fine if there is only one test). +Obviously, libFuzzer may work better if you provide a non-empty +corpus, but fuzzing will work even without an initial corpus, unlike AFL. + +One hint when using libFuzzer is to avoid dynamically allocating +memory during a test, if that memory would not be freed on a test +failure. This will leak memory and libFuzzer will run out of memory +very quickly in each fuzzing session. + +## Fuzzing with AFL + +DeepState can also be used with a file-based fuzzer (e.g. AFL). There are a few steps to this. First, compile DeepState itself with any needed instrumentation. E.g., to use it with AFL, you might want to add something like: @@ -128,7 +160,7 @@ Finally, if an example has more than one test, you need to specify, with a fully qualified name (e.g., `Arithmetic_InvertibleMultiplication_CanFail`), which test to run, using the `--input_which_test` flag to the binary. By -default, DeepState will run the first test defined. +default, DeepState will run the last test defined. You can compile with `afl-clang-fast` and `afl-clang-fast++` for deferred instrumentation. You'll need code like: diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index 0de6ffc..8ca19ee 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -35,7 +35,8 @@ LOG_LEVEL_DEBUG = 0 LOG_LEVEL_INFO = 1 LOG_LEVEL_WARNING = 2 LOG_LEVEL_ERROR = 3 -LOG_LEVEL_FATAL = 4 +LOG_LEVEL_EXTERNAL = 4 +LOG_LEVEL_FATAL = 5 LOGGER = logging.getLogger("deepstate") diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 295d7b5..ab5ec63 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -16,30 +16,93 @@ add_executable(Crash Crash.cpp) target_link_libraries(Crash deepstate) -add_executable(OneOf OneOf.cpp) +if (BUILD_LIBFUZZER) + add_executable(Crash_LF Crash.cpp) + target_link_libraries(Crash_LF deepstate_LF) + target_link_libraries (Crash_LF "-fsanitize=fuzzer") + set_target_properties(Crash_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + +add_executable(OneOf OneOf.cpp) target_link_libraries(OneOf deepstate) +if (BUILD_LIBFUZZER) + add_executable(OneOf_LF OneOf.cpp) + target_link_libraries(OneOf_LF deepstate_LF) + target_link_libraries (OneOf_LF "-fsanitize=fuzzer") + set_target_properties(OneOf_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(Fixture Fixture.cpp) target_link_libraries(Fixture deepstate) +if (BUILD_LIBFUZZER) + add_executable(Fixture_LF Fixture.cpp) + target_link_libraries(Fixture_LF deepstate_LF) + target_link_libraries (Fixture_LF "-fsanitize=fuzzer") + set_target_properties(Fixture_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(Primes Primes.cpp) target_link_libraries(Primes deepstate) +if (BUILD_LIBFUZZER) + add_executable(Primes_LF Primes.cpp) + target_link_libraries(Primes_LF deepstate_LF) + target_link_libraries (Primes_LF "-fsanitize=fuzzer") + set_target_properties(Primes_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(Euler Euler.cpp) target_link_libraries(Euler deepstate) +if (BUILD_LIBFUZZER) + add_executable(Euler_LF Euler.cpp) + target_link_libraries(Euler_LF deepstate_LF) + target_link_libraries (Euler_LF "-fsanitize=fuzzer") + set_target_properties(Euler_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(IntegerOverflow IntegerOverflow.cpp) target_link_libraries(IntegerOverflow deepstate) +if (BUILD_LIBFUZZER) + add_executable(IntegerOverflow_LF IntegerOverflow.cpp) + target_link_libraries(IntegerOverflow_LF deepstate_LF) + target_link_libraries (IntegerOverflow_LF "-fsanitize=fuzzer") + set_target_properties(IntegerOverflow_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(IntegerArithmetic IntegerArithmetic.cpp) target_link_libraries(IntegerArithmetic deepstate) +if (BUILD_LIBFUZZER) + add_executable(IntegerArithmetic_LF IntegerArithmetic.cpp) + target_link_libraries(IntegerArithmetic_LF deepstate_LF) + target_link_libraries (IntegerArithmetic_LF "-fsanitize=fuzzer") + set_target_properties(IntegerArithmetic_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(Lists Lists.cpp) target_link_libraries(Lists deepstate) +if (BUILD_LIBFUZZER) + add_executable(Lists_LF Lists.cpp) + target_link_libraries(Lists_LF deepstate_LF) + target_link_libraries (Lists_LF "-fsanitize=fuzzer") + set_target_properties(Lists_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(StreamingAndFormatting StreamingAndFormatting.cpp) target_link_libraries(StreamingAndFormatting deepstate) +if (BUILD_LIBFUZZER) + add_executable(StreamingAndFormatting_LF StreamingAndFormatting.cpp) + target_link_libraries(StreamingAndFormatting_LF deepstate_LF) + target_link_libraries (StreamingAndFormatting_LF "-fsanitize=fuzzer") + set_target_properties(StreamingAndFormatting_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER") +endif() + add_executable(Squares Squares.c) target_link_libraries(Squares deepstate) set_target_properties(Squares PROPERTIES COMPILE_DEFINITIONS "DEEPSTATE_TEST") diff --git a/examples/Crash.cpp b/examples/Crash.cpp index ca0e66f..7aa1fc5 100644 --- a/examples/Crash.cpp +++ b/examples/Crash.cpp @@ -36,7 +36,9 @@ TEST(Crash, SegFault) { ASSERT_EQ(x, x); } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/examples/Euler.cpp b/examples/Euler.cpp index 20f9790..8f2dd0d 100644 --- a/examples/Euler.cpp +++ b/examples/Euler.cpp @@ -39,7 +39,9 @@ TEST(Euler, SumsOfLikePowers) { << "^5 + " << d << "^5 = " << e << "^5"; } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/examples/Fixture.cpp b/examples/Fixture.cpp index a4af0e1..2c864c6 100644 --- a/examples/Fixture.cpp +++ b/examples/Fixture.cpp @@ -36,8 +36,9 @@ TEST_F(MyTest, Something) { ASSUME_NE(x, 0); } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } - +#endif diff --git a/examples/IntegerArithmetic.cpp b/examples/IntegerArithmetic.cpp index 37b2ac0..7af644d 100644 --- a/examples/IntegerArithmetic.cpp +++ b/examples/IntegerArithmetic.cpp @@ -45,7 +45,9 @@ TEST(Arithmetic, InvertibleMultiplication_CanFail) { }); } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/examples/IntegerOverflow.cpp b/examples/IntegerOverflow.cpp index 09fd32f..c5add45 100644 --- a/examples/IntegerOverflow.cpp +++ b/examples/IntegerOverflow.cpp @@ -40,7 +40,9 @@ TEST(SignedInteger, MultiplicationOverflow) { << x << " squared overflowed."; } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/examples/Lists.cpp b/examples/Lists.cpp index 1e4ad12..8d21f6c 100644 --- a/examples/Lists.cpp +++ b/examples/Lists.cpp @@ -31,7 +31,9 @@ TEST(Vector, DoubleReversal) { }); } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); DeepState_Run(); } +#endif diff --git a/examples/OneOf.cpp b/examples/OneOf.cpp index 1a9bddf..46cd551 100644 --- a/examples/OneOf.cpp +++ b/examples/OneOf.cpp @@ -64,7 +64,9 @@ TEST(OneOfExample, ProduceSixtyOrHigher) { } } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/examples/Primes.cpp b/examples/Primes.cpp index 161527d..1401d35 100644 --- a/examples/Primes.cpp +++ b/examples/Primes.cpp @@ -53,8 +53,9 @@ TEST(PrimePolynomial, OnlyGeneratesPrimes_NoStreaming) { DeepState_Assert(IsPrime(Pump(poly))); } - +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/examples/StreamingAndFormatting.cpp b/examples/StreamingAndFormatting.cpp index f2fb956..f34c8fe 100644 --- a/examples/StreamingAndFormatting.cpp +++ b/examples/StreamingAndFormatting.cpp @@ -40,7 +40,9 @@ TEST(Formatting, OverridePrintf) { printf("hello again!"); } +#ifndef LIBFUZZER int main(int argc, char *argv[]) { DeepState_InitOptions(argc, argv); return DeepState_Run(); } +#endif diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index 8a24179..17dd324 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -491,6 +491,48 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) { } } +/* Run a test case, but in libFuzzer, so not inside a fork. */ +static int DeepState_RunTestLLVM(struct DeepState_TestInfo *test) { + /* Run the test. */ + if (!setjmp(DeepState_ReturnToRun)) { + /* Convert uncaught C++ exceptions into a test failure. */ +#if defined(__cplusplus) && defined(__cpp_exceptions) + try { +#endif /* __cplusplus */ + + test->test_func(); /* Run the test function. */ + return(DeepState_TestRunPass); + +#if defined(__cplusplus) && defined(__cpp_exceptions) + } catch(...) { + DeepState_Fail(); + } +#endif /* __cplusplus */ + + /* We caught a failure when running the test. */ + } else if (DeepState_CatchFail()) { + DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name); + if (HAS_FLAG_output_test_dir) { + DeepState_SaveFailingTest(); + } + return(DeepState_TestRunFail); + + /* The test was abandoned. We may have gotten soft failures before + * abandoning, so we prefer to catch those first. */ + } else if (DeepState_CatchAbandoned()) { + DeepState_LogFormat(DeepState_LogError, "Abandoned: %s", test->test_name); + return(DeepState_TestRunAbandon); + + /* The test passed. */ + } else { + DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name); + if (HAS_FLAG_output_test_dir) { + DeepState_SavePassingTest(); + } + return(DeepState_TestRunPass); + } +} + /* Fork and run `test`. */ static enum DeepState_TestRunResult DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) { @@ -650,7 +692,7 @@ static int DeepState_RunSingleSavedTestDir(void) { } } else { DeepState_LogFormat(DeepState_LogInfo, - "No test specified, defaulting to first test"); + "No test specified, defaulting to last test defined"); break; } } diff --git a/src/include/deepstate/Log.h b/src/include/deepstate/Log.h index 9fb6e14..8cb2767 100644 --- a/src/include/deepstate/Log.h +++ b/src/include/deepstate/Log.h @@ -23,6 +23,8 @@ DEEPSTATE_BEGIN_EXTERN_C +extern int DeepState_UsingLibFuzzer; + struct DeepState_Stream; struct DeepState_VarArgs { @@ -35,8 +37,9 @@ enum DeepState_LogLevel { DeepState_LogWarning = 2, DeepState_LogWarn = DeepState_LogWarning, DeepState_LogError = 3, - DeepState_LogFatal = 4, - DeepState_LogCritical = DeepState_LogFatal + DeepState_LogExternal = 4, + DeepState_LogFatal = 5, + DeepState_LogCritical = DeepState_LogFatal, }; /* Log a C string. */ diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 112cbda..db3a0ed 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -544,6 +544,46 @@ bool DeepState_CatchAbandoned(void) { return DeepState_CurrentTestRun->result == DeepState_TestRunAbandon; } +extern int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + if (Size > sizeof(DeepState_Input)) { + return 0; // Just ignore any too-big inputs + } + + DeepState_UsingLibFuzzer = 1; + + struct DeepState_TestInfo *test = NULL; + + DeepState_InitOptions(0, ""); + //DeepState_Setup(); we want to do our own, simpler, memory management + void *mem = malloc(sizeof(struct DeepState_TestRunInfo)); + DeepState_CurrentTestRun = (struct DeepState_TestRunInfo *) mem; + + test = DeepState_FirstTest(); + const char* which_test = getenv("LIBFUZZER_WHICH_TEST"); + if (which_test != NULL) { + for (test = DeepState_FirstTest(); test != NULL; test = test->prev) { + if (strncmp(which_test, test->test_name, strnlen(which_test, 1024)) == 0) { + break; + } + } + } + + memset((void *) DeepState_Input, 0, sizeof(DeepState_Input)); + DeepState_InputIndex = 0; + + memcpy((void *) DeepState_Input, (void *) Data, Size); + + DeepState_Begin(test); + + enum DeepState_TestRunResult result = DeepState_RunTestLLVM(test); + + DeepState_Teardown(); + DeepState_CurrentTestRun = NULL; + free(mem); + + return 0; // Non-zero return values are reserved for future use. +} + /* Overwrite libc's abort. */ void abort(void) { DeepState_Fail(); diff --git a/src/lib/Log.c b/src/lib/Log.c index c027ac6..cbcb03f 100644 --- a/src/lib/Log.c +++ b/src/lib/Log.c @@ -53,6 +53,8 @@ static const char *DeepState_LogLevelStr(enum DeepState_LogLevel level) { return "WARNING"; case DeepState_LogError: return "ERROR"; + case DeepState_LogExternal: + return "EXTERNAL"; case DeepState_LogFatal: return "FATAL"; default: @@ -64,11 +66,16 @@ enum { DeepState_LogBufSize = 4096 }; +int DeepState_UsingLibFuzzer = 0; + char DeepState_LogBuf[DeepState_LogBufSize + 1] = {}; /* Log a C string. */ DEEPSTATE_NOINLINE void DeepState_Log(enum DeepState_LogLevel level, const char *str) { + if (DeepState_UsingLibFuzzer && (level < DeepState_LogExternal)) { + return; + } memset(DeepState_LogBuf, 0, DeepState_LogBufSize); snprintf(DeepState_LogBuf, DeepState_LogBufSize, "%s: %s\n", DeepState_LogLevelStr(level), str); @@ -90,6 +97,20 @@ void DeepState_LogVFormat(enum DeepState_LogLevel level, const char *format, va_list args) { struct DeepState_VarArgs va; va_copy(va.args, args); + if (DeepState_UsingLibFuzzer && (level < DeepState_LogExternal)) { + return; + } + DeepState_LogStream(level); + DeepState_StreamVFormat(level, format, va.args); + DeepState_LogStream(level); +} + +/* Log some formatted output. */ +DEEPSTATE_NOINLINE +void DeepState_LogVFormatLLVM(enum DeepState_LogLevel level, + const char *format, va_list args) { + struct DeepState_VarArgs va; + va_copy(va.args, args); DeepState_LogStream(level); DeepState_StreamVFormat(level, format, va.args); DeepState_LogStream(level); @@ -149,11 +170,25 @@ int vfprintf(FILE *file, const char *format, va_list args) { } else if (stdout == file) { DeepState_LogVFormat(DeepState_LogInfo, format, args); } else { - DeepState_LogStream(DeepState_LogWarning); - DeepState_Log(DeepState_LogWarning, - "vfprintf with non-stdout/stderr stream follows:"); - DeepState_LogVFormat(DeepState_LogInfo, format, args); + DeepState_LogVFormat(DeepState_LogExternal, format, args); } + /* + Old code. Now let's just log everything with odd dest as "external." + + if (!DeepState_UsingLibFuzzer) { + if (strstr(format, "INFO:") != NULL) { + // Assume such a string to an nonstd target is libFuzzer + DeepState_LogVFormat(DeepState_LogExternal, format, args); + } else { + DeepState_LogStream(DeepState_LogWarning); + DeepState_Log(DeepState_LogWarning, + "vfprintf with non-stdout/stderr stream follows:"); + DeepState_LogVFormat(DeepState_LogInfo, format, args); + } + } else { + DeepState_LogVFormat(DeepState_LogExternal, format, args); + } + */ return 0; }