Merge pull request #89 from trailofbits/support_libfuzzer

Support libFuzzer

CMakeLists.txt

@@ -18,6 +18,12 @@ cmake_minimum_required(VERSION 2.8)
 enable_language(C)
 enable_language(CXX)
 
+set(BUILD_LIBFUZZER "$ENV{BUILD_LIBFUZZER}")
+if (BUILD_LIBFUZZER)
+    SET(CMAKE_C_COMPILER clang)
+    SET(CMAKE_CXX_COMPILER clang++)
+endif()
+
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if(NOT CMAKE_BUILD_TYPE)
@@ -39,7 +45,7 @@ set(CMAKE_CXX_EXTENSIONS ON)
 if (NOT WIN32)
   set(CMAKE_C_STANDARD 99)
   set(CMAKE_CXX_STANDARD 11)
-endif ()
+endif()
 
 add_library(${PROJECT_NAME} STATIC
   src/lib/DeepState.c
@@ -71,7 +77,7 @@ target_include_directories(${PROJECT_NAME}32
   PUBLIC SYSTEM "${CMAKE_SOURCE_DIR}/src/include"
 )
 
-# Install the
+# Install the library
 install(
   DIRECTORY "${CMAKE_SOURCE_DIR}/src/include/deepstate"
   DESTINATION include
@@ -84,6 +90,28 @@ install(
   ARCHIVE DESTINATION lib
 )
 
+if (BUILD_LIBFUZZER)
+    add_library(${PROJECT_NAME}_LF STATIC
+       src/lib/DeepState.c
+       src/lib/Log.c
+       src/lib/Option.c
+       src/lib/Stream.c
+    )
+    
+    target_compile_options(${PROJECT_NAME}_LF PUBLIC -mno-avx -fsanitize=fuzzer-no-link)
+    
+    target_include_directories(${PROJECT_NAME}_LF
+       PUBLIC SYSTEM "${CMAKE_SOURCE_DIR}/src/include"
+    )
+    
+    install(
+       TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_LF
+       LIBRARY DESTINATION lib
+       ARCHIVE DESTINATION lib
+    )
+endif()
+
+
 set(SETUP_PY_IN "${CMAKE_SOURCE_DIR}/bin/setup.py.in")
 set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py")
 configure_file(${SETUP_PY_IN} ${SETUP_PY})

README.md

@@ -12,7 +12,7 @@ The [2018 IEEE Cybersecurity Development Conference](https://secdev.ieee.org/201
 
 * Tests look like Google Test, but can use symbolic execution/fuzzing to generate data (parameterized unit testing)
   * Easier to learn than binary analysis tools/fuzzers, but provides similar functionality
-* Already supports Manticore, Angr, Dr. Fuzz, file-based fuzzing with e.g., AFL; more back-ends likely in future
+* Already supports Manticore, Angr, libFuzzer, file-based fuzzing with e.g., AFL; more back-ends likely in future
   * Switch test generation tool without re-writing test harness
     * Work around show-stopper bugs
     * Find out which tool works best for your code under test
@@ -89,9 +89,41 @@ deepstate-angr --num_workers 4 --output_test_dir out $DEEPSTATE/build/examples/I
 
 DeepState consists of a static library, used to write test harnesses, and command-line _executors_ written in Python. At this time, the best documentation is in the [examples](/examples) and in our [paper](https://agroce.github.io/bar18.pdf).
 
-## Fuzzing
+## Fuzzing with libFuzzer
 
-DeepState now can be used with a file-based fuzzer (e.g. AFL).  There
+If you install clang 6.0 or later, and run `cmake` when you install
+with the `BUILD_LIBFUZZER` environment variable defined, you can
+generate tests using LlibFuzzer.  Because both DeepState and libFuzzer
+want to be `main`, this requires building a different executable for
+libFuzzer.  The `examples` directory shows how this can be done.  The
+libFuzzer executable works like any other libFuzzer executable, and
+the tests produced can be run using the normal DeepState executable.
+For example, generating some tests of the `OneOf` example (up to 5,000
+runs), then running those tests to examine the results, would look
+like:
+
+```shell
+mkdir OneOf_libFuzzer_corpus
+./OneOf_LF -runs=5000 OneOf_libFuzzer_corpus
+./OneOf --input_test_files_dir OneOf_libFuzzer_corpus
+```
+
+Use the `LIBFUZZER_WHICH_TEST`
+environment variable to control which test libFuzzer runs, using a
+fully qualified name (e.g.,
+`Arithmetic_InvertibleMultiplication_CanFail`).  By default, you get
+the last test defined (which works fine if there is only one test).
+Obviously, libFuzzer may work better if you provide a non-empty
+corpus, but fuzzing will work even without an initial corpus, unlike AFL.
+
+One hint when using libFuzzer is to avoid dynamically allocating
+memory during a test, if that memory would not be freed on a test
+failure.  This will leak memory and libFuzzer will run out of memory
+very quickly in each fuzzing session.
+
+## Fuzzing with AFL
+
+DeepState can also be used with a file-based fuzzer (e.g. AFL).  There
 are a few steps to this.  First, compile DeepState itself with any
 needed instrumentation.  E.g., to use it with AFL, you might want to add
 something like:
@@ -128,7 +160,7 @@ Finally, if an example has more than one test, you need to specify,
 with a fully qualified name (e.g.,
 `Arithmetic_InvertibleMultiplication_CanFail`), which test to run,
 using the `--input_which_test` flag to the binary.  By
-default, DeepState will run the first test defined.
+default, DeepState will run the last test defined.
 
 You can compile with `afl-clang-fast` and `afl-clang-fast++` for
 deferred instrumentation.  You'll need code like:

bin/deepstate/common.py

@@ -35,7 +35,8 @@ LOG_LEVEL_DEBUG = 0
 LOG_LEVEL_INFO = 1
 LOG_LEVEL_WARNING = 2
 LOG_LEVEL_ERROR = 3
-LOG_LEVEL_FATAL = 4
+LOG_LEVEL_EXTERNAL = 4
+LOG_LEVEL_FATAL = 5
 
 
 LOGGER = logging.getLogger("deepstate")

examples/CMakeLists.txt

@@ -16,30 +16,93 @@
 add_executable(Crash Crash.cpp)
 target_link_libraries(Crash deepstate)
 
-add_executable(OneOf OneOf.cpp)
+if (BUILD_LIBFUZZER)
+   add_executable(Crash_LF Crash.cpp) 
+   target_link_libraries(Crash_LF deepstate_LF)
+   target_link_libraries (Crash_LF "-fsanitize=fuzzer")
+   set_target_properties(Crash_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
+add_executable(OneOf OneOf.cpp) 
 target_link_libraries(OneOf deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(OneOf_LF OneOf.cpp) 
+   target_link_libraries(OneOf_LF deepstate_LF)
+   target_link_libraries (OneOf_LF "-fsanitize=fuzzer")
+   set_target_properties(OneOf_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(Fixture Fixture.cpp)
 target_link_libraries(Fixture deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(Fixture_LF Fixture.cpp) 
+   target_link_libraries(Fixture_LF deepstate_LF)
+   target_link_libraries (Fixture_LF "-fsanitize=fuzzer")
+   set_target_properties(Fixture_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(Primes Primes.cpp)
 target_link_libraries(Primes deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(Primes_LF Primes.cpp) 
+   target_link_libraries(Primes_LF deepstate_LF)
+   target_link_libraries (Primes_LF "-fsanitize=fuzzer")
+   set_target_properties(Primes_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(Euler Euler.cpp)
 target_link_libraries(Euler deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(Euler_LF Euler.cpp) 
+   target_link_libraries(Euler_LF deepstate_LF)
+   target_link_libraries (Euler_LF "-fsanitize=fuzzer")
+   set_target_properties(Euler_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(IntegerOverflow IntegerOverflow.cpp)
 target_link_libraries(IntegerOverflow deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(IntegerOverflow_LF IntegerOverflow.cpp) 
+   target_link_libraries(IntegerOverflow_LF deepstate_LF)
+   target_link_libraries (IntegerOverflow_LF "-fsanitize=fuzzer")
+   set_target_properties(IntegerOverflow_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(IntegerArithmetic IntegerArithmetic.cpp)
 target_link_libraries(IntegerArithmetic deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(IntegerArithmetic_LF IntegerArithmetic.cpp) 
+   target_link_libraries(IntegerArithmetic_LF deepstate_LF)
+   target_link_libraries (IntegerArithmetic_LF "-fsanitize=fuzzer")
+   set_target_properties(IntegerArithmetic_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(Lists Lists.cpp)
 target_link_libraries(Lists deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(Lists_LF Lists.cpp) 
+   target_link_libraries(Lists_LF deepstate_LF)
+   target_link_libraries (Lists_LF "-fsanitize=fuzzer")
+   set_target_properties(Lists_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(StreamingAndFormatting StreamingAndFormatting.cpp)
 target_link_libraries(StreamingAndFormatting deepstate)
 
+if (BUILD_LIBFUZZER)
+   add_executable(StreamingAndFormatting_LF StreamingAndFormatting.cpp) 
+   target_link_libraries(StreamingAndFormatting_LF deepstate_LF)
+   target_link_libraries (StreamingAndFormatting_LF "-fsanitize=fuzzer")
+   set_target_properties(StreamingAndFormatting_LF PROPERTIES COMPILE_DEFINITIONS "LIBFUZZER")
+endif()
+
 add_executable(Squares Squares.c)
 target_link_libraries(Squares deepstate)
 set_target_properties(Squares PROPERTIES COMPILE_DEFINITIONS "DEEPSTATE_TEST")

examples/Crash.cpp

@@ -36,7 +36,9 @@ TEST(Crash, SegFault) {
   ASSERT_EQ(x, x);
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

examples/Euler.cpp

@@ -39,7 +39,9 @@ TEST(Euler, SumsOfLikePowers) {
       << "^5 + " << d << "^5 = " << e << "^5";
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

examples/Fixture.cpp

@@ -36,8 +36,9 @@ TEST_F(MyTest, Something) {
   ASSUME_NE(x, 0);
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
-
+#endif

examples/IntegerArithmetic.cpp

@@ -45,7 +45,9 @@ TEST(Arithmetic, InvertibleMultiplication_CanFail) {
   });
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

examples/IntegerOverflow.cpp

@@ -40,7 +40,9 @@ TEST(SignedInteger, MultiplicationOverflow) {
       << x << " squared overflowed.";
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

examples/Lists.cpp

@@ -31,7 +31,9 @@ TEST(Vector, DoubleReversal) {
   });
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   DeepState_Run();
 }
+#endif

examples/OneOf.cpp

@@ -64,7 +64,9 @@ TEST(OneOfExample, ProduceSixtyOrHigher) {
   }
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

examples/Primes.cpp

@@ -53,8 +53,9 @@ TEST(PrimePolynomial, OnlyGeneratesPrimes_NoStreaming) {
   DeepState_Assert(IsPrime(Pump(poly)));
 }
 
-
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

examples/StreamingAndFormatting.cpp

@@ -40,7 +40,9 @@ TEST(Formatting, OverridePrintf) {
   printf("hello again!");
 }
 
+#ifndef LIBFUZZER
 int main(int argc, char *argv[]) {
   DeepState_InitOptions(argc, argv);
   return DeepState_Run();
 }
+#endif

src/include/deepstate/DeepState.h

@@ -491,6 +491,48 @@ static void DeepState_RunTest(struct DeepState_TestInfo *test) {
   }
 }
 
+/* Run a test case, but in libFuzzer, so not inside a fork. */
+static int DeepState_RunTestLLVM(struct DeepState_TestInfo *test) {
+  /* Run the test. */
+  if (!setjmp(DeepState_ReturnToRun)) {
+    /* Convert uncaught C++ exceptions into a test failure. */
+#if defined(__cplusplus) && defined(__cpp_exceptions)
+    try {
+#endif  /* __cplusplus */
+
+      test->test_func();  /* Run the test function. */
+      return(DeepState_TestRunPass);
+
+#if defined(__cplusplus) && defined(__cpp_exceptions)
+    } catch(...) {
+      DeepState_Fail();
+    }
+#endif  /* __cplusplus */
+
+    /* We caught a failure when running the test. */
+  } else if (DeepState_CatchFail()) {
+    DeepState_LogFormat(DeepState_LogError, "Failed: %s", test->test_name);
+    if (HAS_FLAG_output_test_dir) {
+      DeepState_SaveFailingTest();
+    }
+    return(DeepState_TestRunFail);
+
+    /* The test was abandoned. We may have gotten soft failures before
+     * abandoning, so we prefer to catch those first. */
+  } else if (DeepState_CatchAbandoned()) {
+    DeepState_LogFormat(DeepState_LogError, "Abandoned: %s", test->test_name);
+    return(DeepState_TestRunAbandon);
+
+    /* The test passed. */
+  } else {
+    DeepState_LogFormat(DeepState_LogInfo, "Passed: %s", test->test_name);
+    if (HAS_FLAG_output_test_dir) {
+      DeepState_SavePassingTest();
+    }
+    return(DeepState_TestRunPass);
+  }
+}
+
 /* Fork and run `test`. */
 static enum DeepState_TestRunResult
 DeepState_ForkAndRunTest(struct DeepState_TestInfo *test) {
@@ -650,7 +692,7 @@ static int DeepState_RunSingleSavedTestDir(void) {
       }
     } else {
       DeepState_LogFormat(DeepState_LogInfo,
-			  "No test specified, defaulting to first test");
+			  "No test specified, defaulting to last test defined");
       break;
     }
   }

src/include/deepstate/Log.h

@@ -23,6 +23,8 @@
 
 DEEPSTATE_BEGIN_EXTERN_C
 
+extern int DeepState_UsingLibFuzzer;
+
 struct DeepState_Stream;
 
 struct DeepState_VarArgs {
@@ -35,8 +37,9 @@ enum DeepState_LogLevel {
   DeepState_LogWarning = 2,
   DeepState_LogWarn = DeepState_LogWarning,
   DeepState_LogError = 3,
-  DeepState_LogFatal = 4,
-  DeepState_LogCritical = DeepState_LogFatal
+  DeepState_LogExternal = 4,
+  DeepState_LogFatal = 5,
+  DeepState_LogCritical = DeepState_LogFatal,
 };
 
 /* Log a C string. */

src/lib/DeepState.c

@@ -544,6 +544,46 @@ bool DeepState_CatchAbandoned(void) {
   return DeepState_CurrentTestRun->result == DeepState_TestRunAbandon;
 }
 
+extern int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size > sizeof(DeepState_Input)) {
+    return 0; // Just ignore any too-big inputs
+  }
+
+  DeepState_UsingLibFuzzer = 1;
+  
+  struct DeepState_TestInfo *test = NULL;
+
+  DeepState_InitOptions(0, "");  
+  //DeepState_Setup(); we want to do our own, simpler, memory management
+  void *mem = malloc(sizeof(struct DeepState_TestRunInfo));
+  DeepState_CurrentTestRun = (struct DeepState_TestRunInfo *) mem;
+
+  test = DeepState_FirstTest();
+  const char* which_test = getenv("LIBFUZZER_WHICH_TEST");
+  if (which_test != NULL) {
+    for (test = DeepState_FirstTest(); test != NULL; test = test->prev) {
+      if (strncmp(which_test, test->test_name, strnlen(which_test, 1024)) == 0) {
+	break;
+      }
+    }
+  }
+
+  memset((void *) DeepState_Input, 0, sizeof(DeepState_Input));
+  DeepState_InputIndex = 0;
+
+  memcpy((void *) DeepState_Input, (void *) Data, Size);
+
+  DeepState_Begin(test);
+
+  enum DeepState_TestRunResult result = DeepState_RunTestLLVM(test);
+
+  DeepState_Teardown();
+  DeepState_CurrentTestRun = NULL;
+  free(mem);
+  
+  return 0;  // Non-zero return values are reserved for future use.
+}
+
 /* Overwrite libc's abort. */
 void abort(void) {
   DeepState_Fail();

src/lib/Log.c

@@ -53,6 +53,8 @@ static const char *DeepState_LogLevelStr(enum DeepState_LogLevel level) {
       return "WARNING";
     case DeepState_LogError:
       return "ERROR";
+    case DeepState_LogExternal:
+      return "EXTERNAL";
     case DeepState_LogFatal:
       return "FATAL";
     default:
@@ -64,11 +66,16 @@ enum {
   DeepState_LogBufSize = 4096
 };
 
+int DeepState_UsingLibFuzzer = 0;
+
 char DeepState_LogBuf[DeepState_LogBufSize + 1] = {};
 
 /* Log a C string. */
 DEEPSTATE_NOINLINE
 void DeepState_Log(enum DeepState_LogLevel level, const char *str) {
+  if (DeepState_UsingLibFuzzer && (level < DeepState_LogExternal)) {
+    return;
+  }
   memset(DeepState_LogBuf, 0, DeepState_LogBufSize);
   snprintf(DeepState_LogBuf, DeepState_LogBufSize, "%s: %s\n",
            DeepState_LogLevelStr(level), str);
@@ -90,6 +97,20 @@ void DeepState_LogVFormat(enum DeepState_LogLevel level,
                           const char *format, va_list args) {
   struct DeepState_VarArgs va;
   va_copy(va.args, args);
+  if (DeepState_UsingLibFuzzer && (level < DeepState_LogExternal)) {
+    return;
+  }
+  DeepState_LogStream(level);
+  DeepState_StreamVFormat(level, format, va.args);
+  DeepState_LogStream(level);
+}
+
+/* Log some formatted output. */
+DEEPSTATE_NOINLINE
+void DeepState_LogVFormatLLVM(enum DeepState_LogLevel level,
+			      const char *format, va_list args) {
+  struct DeepState_VarArgs va;
+  va_copy(va.args, args);
   DeepState_LogStream(level);
   DeepState_StreamVFormat(level, format, va.args);
   DeepState_LogStream(level);
@@ -149,11 +170,25 @@ int vfprintf(FILE *file, const char *format, va_list args) {
   } else if (stdout == file) {
     DeepState_LogVFormat(DeepState_LogInfo, format, args);
   } else {
-    DeepState_LogStream(DeepState_LogWarning);
-    DeepState_Log(DeepState_LogWarning,
-                  "vfprintf with non-stdout/stderr stream follows:");
-    DeepState_LogVFormat(DeepState_LogInfo, format, args);
+    DeepState_LogVFormat(DeepState_LogExternal, format, args);
   }
+  /*
+    Old code.  Now let's just log everything with odd dest as "external."
+
+    if (!DeepState_UsingLibFuzzer) {
+      if (strstr(format, "INFO:") != NULL) {
+	// Assume such a string to an nonstd target is libFuzzer
+	DeepState_LogVFormat(DeepState_LogExternal, format, args);
+      } else {
+	DeepState_LogStream(DeepState_LogWarning);
+	DeepState_Log(DeepState_LogWarning,
+		      "vfprintf with non-stdout/stderr stream follows:");
+	DeepState_LogVFormat(DeepState_LogInfo, format, args);
+      }
+    } else {
+      DeepState_LogVFormat(DeepState_LogExternal, format, args);      
+    }
+  */
   return 0;
 }