diff --git a/README.md b/README.md index 980681b..5d11805 100644 --- a/README.md +++ b/README.md @@ -115,14 +115,142 @@ argument to see all DeepState options. DeepState consists of a static library, used to write test harnesses, and command-line _executors_ written in Python. At this time, the best documentation is in the [examples](/examples) and in our [paper](https://agroce.github.io/bar18.pdf). A more extensive example, using DeepState and libFuzzer to test a user-mode file system, is available [here](https://github.com/agroce/testfs); in particular the [Tests.cpp](https://github.com/agroce/testfs/blob/master/Tests.cpp) file and CMakeLists.txt show DeepState usage. +## Example Code + +``` +#include + +using namespace deepstate; + +/* Simple, buggy, run-length encoding that creates "human readable" + * encodings by adding 'A'-1 to the count, and splitting at 26. + * e.g., encode("aaabbbbbc") = "aCbEcA" since C=3 and E=5 */ + +char* encode(const char* input) { + unsigned int len = strlen(input); + char* encoded = (char*)malloc((len*2)+1); + int pos = 0; + if (len > 0) { + unsigned char last = input[0]; + int count = 1; + for (int i = 1; i < len; i++) { + if (((unsigned char)input[i] == last) && (count < 26)) + count++; + else { + encoded[pos++] = last; + encoded[pos++] = 64 + count; + last = (unsigned char)input[i]; + count = 1; + } + } + encoded[pos++] = last; + encoded[pos++] = 65; // Should be 64 + count + } + encoded[pos] = '\0'; + return encoded; +} + +char* decode(const char* output) { + unsigned int len = strlen(output); + char* decoded = (char*)malloc((len/2)*26); + int pos = 0; + for (int i = 0; i < len; i += 2) { + for (int j = 0; j < (output[i+1] - 64); j++) { + decoded[pos++] = output[i]; + } + } + decoded[pos] = '\0'; + return decoded; +} + +// Can be (much) higher (e.g., > 1024) if we're using fuzzing, not symbolic execution +#define MAX_STR_LEN 6 + +TEST(Runlength, BoringUnitTest) { + ASSERT_EQ(strcmp(encode(""), ""), 0); + ASSERT_EQ(strcmp(encode("a"), "aA"), 0); + ASSERT_EQ(strcmp(encode("aaabbbbbc"), "aCbEcA"), 0); +} + +TEST(Runlength, EncodeDecode) { + char* original = DeepState_CStrUpToLen(MAX_STR_LEN, "abcdef0123456789"); + char* encoded = encode(original); + ASSERT_LE(strlen(encoded), strlen(original)*2) << "Encoding is > length*2!"; + char* roundtrip = decode(encoded); + ASSERT_EQ(strncmp(roundtrip, original, MAX_STR_LEN), 0) << + "ORIGINAL: '" << original << "', ENCODED: '" << encoded << + "', ROUNDTRIP: '" << roundtrip << "'"; +} +``` + +The code above (which can be found +[here](https://github.com/trailofbits/deepstate/blob/master/examples/Runlen.cpp)) +shows an example of a DeepState test harness. Most of the code is +just the functions to be tested. Using DeepState to test them requires: + +- Including the DeepState C++ header and using the DeepState namespace + +- Defining at least one TEST, with names + +- Calling some DeepState APIs that produce data + - In this example, we see the `DeepState_CStrUpToLen` call tells + DeepState to produce a string that has up the `MAX_STR_LEN` + characters, chosen from those present in hex strings. + +- Optionally making some assertions about the correctness of the +results + - In `Runlen.cpp` this is the `ASSERT_LE` and `ASSERT_EQ` checks. + - In the absence of any properties to check, DeepState can still + look for memory safety violations, crashes, and other general + categories of undesirable behavior, like any fuzzer. + +DeepState will also run the "BoringUnitTest," but it (like a +traditional hand-written unit test) is simply a test of fixed inputs +devised by a programmer. These inputs do not expose the bug in +`encode`. Nor do the default values for the DeepState test: + +``` +~/deepstate/build/examples$ ./Runlen +TRACE: Running: Runlength_EncodeDecode from /Users/alex/deepstate/examples/Runlen.cpp(55) +TRACE: Passed: Runlength_EncodeDecode +TRACE: Running: Runlength_BoringUnitTest from /Users/alex/deepstate/examples/Runlen.cpp(49) +TRACE: Passed: Runlength_BoringUnitTest +``` + +Using DeepState, however, it is easy to find the bug. Just +go into the `$DEEPSTATE/build/examples` directory and try: + +```shell +deepstate-angr ./Runlen +``` + +or + +```shell +./Runlen --fuzz --abort_on_fail +``` + +The fuzzer will output something like: + +``` +INFO: Starting fuzzing +WARNING: No seed provided; using 1546631311 +WARNING: No test specified, defaulting to last test defined (Runlength_EncodeDecode) +CRITICAL: /Users/alex/deepstate/examples/Runlen.cpp(60): ORIGINAL: '91c499', ENCODED: '9A1AcA4A9A', ROUNDTRIP: '91c49' +ERROR: Failed: Runlength_EncodeDecode +``` + ## Log Levels By default, DeepState is not very verbose about testing activity, -other than failing tests. The `--log-level` argument lowers the +other than failing tests. The `--log_level` argument lowers the threshold for output, with 0 = `DEBUG`, 1 = `TRACE` (output from the tests, including from `printf`), 2 = INFO (DeepState messages, the default), 3 = `WARNING`, 4 = `ERROR`, 5 = `EXTERNAL` (output from other programs such as -libFuzzer), and 6 = `CRITICAL` messages. +libFuzzer), and 6 = `CRITICAL` messages. Lowering the `log_level` can be very +useful for understanding what a DeepState harness is actually doing; +often, setting `--log_level 1` in either fuzzing or symbolic +execution will give sufficient information to debug your test harness. ## A Note on Mac OS and Forking @@ -139,15 +267,27 @@ Every DeepState executable provides a simple built-in fuzzer that generates tests using completely random data. Using this fuzzer is as simple as calling the native executable with the `--fuzz` argument. The fuzzer also takes a `seed` and `timeout` (default of two minutes) -to control the fuzzing. If you want to actually save the test cases +to control the fuzzing. By default fuzzing saves +only failing and crashing tests, and these only when given an output +directory. If you want to actually save the test cases generated, you need to add a `--output_test_dir` argument to tell -DeepState where to put the generated tests. By default fuzzing saves -only failing and crashing tests and only when given an output directory. +DeepState where to put the generated tests, and if you want the +(totally random and unlikely to be high-quality) passing tests, you +need to add `--fuzz_save_passing`. Note that while symbolic execution only works on Linux, without a -fairly complex cross-compliation process, the brute force fuzzer works +fairly complex cross-compilation process, the brute force fuzzer works on macOS or (as far as we know) any Unix-like system. +## A Note on Mac OS and Forking + +Normally, when running a test for replay or fuzzing, DeepState forks +in order to cleanly handle crashes of a test. Unfortunately, `fork()` +on mac OS is _extremely_ slow. When using the built-in fuzzer or +replaying more than a few tests, it is highly recommended to add the `--no_fork` +option on mac OS, unless you need the added crash handling (that is, +only when things aren't working without that option). + ## Fuzzing with libFuzzer If you install clang 6.0 or later, and run `cmake` when you install diff --git a/examples/Runlen.cpp b/examples/Runlen.cpp index b7d12aa..b1d8332 100644 --- a/examples/Runlen.cpp +++ b/examples/Runlen.cpp @@ -3,23 +3,28 @@ using namespace deepstate; /* Simple, buggy, run-length encoding that creates "human readable" - * encodings by adding 'A'-1 to the count, and splitting at 26 */ + * encodings by adding 'A'-1 to the count, and splitting at 26. + * e.g., encode("aaabbbbbc") = "aCbEcA" since C=3 and E=5 */ char* encode(const char* input) { unsigned int len = strlen(input); char* encoded = (char*)malloc((len*2)+1); int pos = 0; - if (strlen(input) > 0) { - unsigned char last = input[0]; int count = 1; + if (len > 0) { + unsigned char last = input[0]; + int count = 1; for (int i = 1; i < len; i++) { if (((unsigned char)input[i] == last) && (count < 26)) count++; else { - encoded[pos++] = last; encoded[pos++] = 64 + count; - last = (unsigned char)input[i]; count = 1; + encoded[pos++] = last; + encoded[pos++] = 64 + count; + last = (unsigned char)input[i]; + count = 1; } } - encoded[pos++] = last; encoded[pos++] = 65; // Should be 64 + count + encoded[pos++] = last; + encoded[pos++] = 65; // Should be 64 + count } encoded[pos] = '\0'; return encoded; @@ -29,35 +34,30 @@ char* decode(const char* output) { unsigned int len = strlen(output); char* decoded = (char*)malloc((len/2)*26); int pos = 0; - if (strlen(output) > 0) { - for (int i = 0; i < len; i += 2) - for (int j = 0; j < (output[i+1] - 64); j++) - decoded[pos++] = output[i]; + for (int i = 0; i < len; i += 2) { + for (int j = 0; j < (output[i+1] - 64); j++) { + decoded[pos++] = output[i]; + } } decoded[pos] = '\0'; return decoded; } -void printBytes(const char* bytes) { - unsigned int len = strlen(bytes); - for (int i = 0; i < len; i++) - LOG(ERROR) << "[" << i << "] = " << (unsigned int)(unsigned char)bytes[i]; -} - // Can be (much) higher (e.g., > 1024) if we're using fuzzing, not symbolic execution #define MAX_STR_LEN 6 -TEST(Runlength, EncodeDecode) { - char* original = DeepState_CStrUpToLen(MAX_STR_LEN); - char* encoded = encode(original); - char* roundtrip = decode(encoded); - if (!(strncmp(roundtrip, original, MAX_STR_LEN) == 0)) { - LOG(ERROR) << "ORIGINAL:"; - printBytes(original); - LOG(ERROR) << "ENCODED:"; - printBytes(encoded); - LOG(ERROR) << "ROUNDTRIP:"; - printBytes(roundtrip); - ASSERT (0) << "Round trip check failed"; - } +TEST(Runlength, BoringUnitTest) { + ASSERT_EQ(strcmp(encode(""), ""), 0); + ASSERT_EQ(strcmp(encode("a"), "aA"), 0); + ASSERT_EQ(strcmp(encode("aaabbbbbc"), "aCbEcA"), 0); +} + +TEST(Runlength, EncodeDecode) { + char* original = DeepState_CStrUpToLen(MAX_STR_LEN, "abcdef0123456789"); + char* encoded = encode(original); + ASSERT_LE(strlen(encoded), strlen(original)*2) << "Encoding is > length*2!"; + char* roundtrip = decode(encoded); + ASSERT_EQ(strncmp(roundtrip, original, MAX_STR_LEN), 0) << + "ORIGINAL: '" << original << "', ENCODED: '" << encoded << + "', ROUNDTRIP: '" << roundtrip << "'"; }