From 59a3b51f9f2f3bba31d201304298c1535bd18f17 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 11:44:33 -0700 Subject: [PATCH 01/10] Tidy up the example --- examples/Runlen.cpp | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/examples/Runlen.cpp b/examples/Runlen.cpp index b7d12aa..9ec722e 100644 --- a/examples/Runlen.cpp +++ b/examples/Runlen.cpp @@ -38,26 +38,15 @@ char* decode(const char* output) { return decoded; } -void printBytes(const char* bytes) { - unsigned int len = strlen(bytes); - for (int i = 0; i < len; i++) - LOG(ERROR) << "[" << i << "] = " << (unsigned int)(unsigned char)bytes[i]; -} - // Can be (much) higher (e.g., > 1024) if we're using fuzzing, not symbolic execution #define MAX_STR_LEN 6 TEST(Runlength, EncodeDecode) { - char* original = DeepState_CStrUpToLen(MAX_STR_LEN); + char* original = DeepState_CStrUpToLen(MAX_STR_LEN, "abcdef0123456789"); char* encoded = encode(original); + ASSERT_LE(strlen(encoded), strlen(original)*2) << "Encoding is > length*2!"; char* roundtrip = decode(encoded); - if (!(strncmp(roundtrip, original, MAX_STR_LEN) == 0)) { - LOG(ERROR) << "ORIGINAL:"; - printBytes(original); - LOG(ERROR) << "ENCODED:"; - printBytes(encoded); - LOG(ERROR) << "ROUNDTRIP:"; - printBytes(roundtrip); - ASSERT (0) << "Round trip check failed"; - } + ASSERT_EQ(strncmp(roundtrip, original, MAX_STR_LEN), 0) << + "ORIGINAL: '" << original << "', ENCODED: '" << encoded << + "', ROUNDTRIP: '" << roundtrip << "'"; } From 276ce6cb3f1fd8768c9446e21f511ac50f2008aa Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 12:00:57 -0700 Subject: [PATCH 02/10] clean up formatting --- examples/Runlen.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/Runlen.cpp b/examples/Runlen.cpp index 9ec722e..10264b2 100644 --- a/examples/Runlen.cpp +++ b/examples/Runlen.cpp @@ -9,17 +9,21 @@ char* encode(const char* input) { unsigned int len = strlen(input); char* encoded = (char*)malloc((len*2)+1); int pos = 0; - if (strlen(input) > 0) { - unsigned char last = input[0]; int count = 1; + if (len > 0) { + unsigned char last = input[0]; + int count = 1; for (int i = 1; i < len; i++) { if (((unsigned char)input[i] == last) && (count < 26)) count++; else { - encoded[pos++] = last; encoded[pos++] = 64 + count; - last = (unsigned char)input[i]; count = 1; + encoded[pos++] = last; + encoded[pos++] = 64 + count; + last = (unsigned char)input[i]; + count = 1; } } - encoded[pos++] = last; encoded[pos++] = 65; // Should be 64 + count + encoded[pos++] = last; + encoded[pos++] = 65; // Should be 64 + count } encoded[pos] = '\0'; return encoded; @@ -29,10 +33,10 @@ char* decode(const char* output) { unsigned int len = strlen(output); char* decoded = (char*)malloc((len/2)*26); int pos = 0; - if (strlen(output) > 0) { - for (int i = 0; i < len; i += 2) - for (int j = 0; j < (output[i+1] - 64); j++) - decoded[pos++] = output[i]; + for (int i = 0; i < len; i += 2) { + for (int j = 0; j < (output[i+1] - 64); j++) { + decoded[pos++] = output[i]; + } } decoded[pos] = '\0'; return decoded; From 65e86bcc8200f5e3327a53e5ea5817b93a34b2f9 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 12:19:27 -0700 Subject: [PATCH 03/10] add code in README --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/README.md b/README.md index 82d41a2..b202f85 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,67 @@ replaying tests, it is highly recommended to add the `--no_fork` option on mac OS, unless you need the added crash handling (that is, things aren't working without that option). +## Example Code + +``` +#include + +using namespace deepstate; + +/* Simple, buggy, run-length encoding that creates "human readable" + * encodings by adding 'A'-1 to the count, and splitting at 26 */ + +char* encode(const char* input) { + unsigned int len = strlen(input); + char* encoded = (char*)malloc((len*2)+1); + int pos = 0; + if (len > 0) { + unsigned char last = input[0]; + int count = 1; + for (int i = 1; i < len; i++) { + if (((unsigned char)input[i] == last) && (count < 26)) + count++; + else { + encoded[pos++] = last; + encoded[pos++] = 64 + count; + last = (unsigned char)input[i]; + count = 1; + } + } + encoded[pos++] = last; + encoded[pos++] = 65; // Should be 64 + count + } + encoded[pos] = '\0'; + return encoded; +} + +char* decode(const char* output) { + unsigned int len = strlen(output); + char* decoded = (char*)malloc((len/2)*26); + int pos = 0; + for (int i = 0; i < len; i += 2) { + for (int j = 0; j < (output[i+1] - 64); j++) { + decoded[pos++] = output[i]; + } + } + decoded[pos] = '\0'; + return decoded; +} + +// Can be (much) higher (e.g., > 1024) if we're using fuzzing, not symbolic execution +#define MAX_STR_LEN 6 + +TEST(Runlength, EncodeDecode) { + char* original = DeepState_CStrUpToLen(MAX_STR_LEN, "abcdef0123456789"); + char* encoded = encode(original); + ASSERT_LE(strlen(encoded), strlen(original)*2) << "Encoding is > length*2!"; + char* roundtrip = decode(encoded); + ASSERT_EQ(strncmp(roundtrip, original, MAX_STR_LEN), 0) << + "ORIGINAL: '" << original << "', ENCODED: '" << encoded << + "', ROUNDTRIP: '" << roundtrip << "'"; +} +``` + ## Built-In Fuzzer Every DeepState executable provides a simple built-in fuzzer that From f55bad722a8c0a4c9d504d9f95da3236b09d2b12 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 12:25:31 -0700 Subject: [PATCH 04/10] comment on the code --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index b202f85..0323f60 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,27 @@ TEST(Runlength, EncodeDecode) { } ``` +The code above (which can be found +[here](https://github.com/trailofbits/deepstate/blob/master/examples/Runlen.cpp) +shows an example of a DeepState test harness. Most of the code is +just the functions to be tested. Using DeepState to test them requires: + +- including the DeepState C++ header and using the DeepState namespace + +- defining at least one TEST, with names + +- calling some DeepState APIs that produce data + - in this example, we see the `DeepState_CStrUpToLen` call tells + DeepState to produce a string that has up the `MAX_STR_LEN` + characters, chosen from those present in hex strings. + +- optionally making some assertions about the correctness of the +results + - in this example, the `ASSERT_LE` and `ASSERT_EQ` checks + - in the absence of any properties to check, DeepState can still + look for memory safety violations, crashes, and other general + categories of undesireable behavior, like any fuzzer + ## Built-In Fuzzer Every DeepState executable provides a simple built-in fuzzer that From 0c8185567d8f0c9353687496224a5f6d8ea0b2b7 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 12:35:06 -0700 Subject: [PATCH 05/10] comment clarifying example --- README.md | 3 ++- examples/Runlen.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0323f60..f55104a 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,8 @@ things aren't working without that option). using namespace deepstate; /* Simple, buggy, run-length encoding that creates "human readable" - * encodings by adding 'A'-1 to the count, and splitting at 26 */ + * encodings by adding 'A'-1 to the count, and splitting at 26. + * e.g., encode("aaabbbbbc") = "aCbEcA" since C=3 and E=5 */ char* encode(const char* input) { unsigned int len = strlen(input); diff --git a/examples/Runlen.cpp b/examples/Runlen.cpp index 10264b2..2b2396c 100644 --- a/examples/Runlen.cpp +++ b/examples/Runlen.cpp @@ -3,7 +3,8 @@ using namespace deepstate; /* Simple, buggy, run-length encoding that creates "human readable" - * encodings by adding 'A'-1 to the count, and splitting at 26 */ + * encodings by adding 'A'-1 to the count, and splitting at 26. + * e.g., encode("aaabbbbbc") = "aCbEcA" since C=3 and E=5 */ char* encode(const char* input) { unsigned int len = strlen(input); From 878528e672466202ff5453b6e659f82ff270d8fe Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 12:49:43 -0700 Subject: [PATCH 06/10] add boring unit test --- README.md | 32 ++++++++++++++++++++++++++++++++ examples/Runlen.cpp | 6 ++++++ 2 files changed, 38 insertions(+) diff --git a/README.md b/README.md index f55104a..ff775b1 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,12 @@ char* decode(const char* output) { // Can be (much) higher (e.g., > 1024) if we're using fuzzing, not symbolic execution #define MAX_STR_LEN 6 +TEST(Runlength, BoringUnitTest) { + ASSERT_EQ(strcmp(encode(""), ""), 0); + ASSERT_EQ(strcmp(encode("a"), "aA"), 0); + ASSERT_EQ(strcmp(encode("aaabbbbbc"), "aCbEcA"), 0); +} + TEST(Runlength, EncodeDecode) { char* original = DeepState_CStrUpToLen(MAX_STR_LEN, "abcdef0123456789"); char* encoded = encode(original); @@ -216,6 +222,32 @@ results look for memory safety violations, crashes, and other general categories of undesireable behavior, like any fuzzer +DeepState will also run the "BoringUnitTest," but it (like a +traditional hand-written unit test) is simply a test of fixed inputs +devised by a programmer. These inputs do not expose the bug in +`encode`. Using DeepState, however, it is easy to find the bug. Just +go into the `$DEEPSTATE/build/examples` directory and try: + +```shell +deepstate-angr ./Runlen +``` + +or + +```shell +./Runlen --fuzz --abort_on_fail +``` + +The fuzzer will output something like: + +``` +INFO: Starting fuzzing +WARNING: No seed provided; using 1546631311 +WARNING: No test specified, defaulting to last test defined (Runlength_EncodeDecode) +CRITICAL: /Users/alex/deepstate/examples/Runlen.cpp(60): ORIGINAL: '91c499', ENCODED: '9A1AcA4A9A', ROUNDTRIP: '91c49' +ERROR: Failed: Runlength_EncodeDecode +``` + ## Built-In Fuzzer Every DeepState executable provides a simple built-in fuzzer that diff --git a/examples/Runlen.cpp b/examples/Runlen.cpp index 2b2396c..b1d8332 100644 --- a/examples/Runlen.cpp +++ b/examples/Runlen.cpp @@ -46,6 +46,12 @@ char* decode(const char* output) { // Can be (much) higher (e.g., > 1024) if we're using fuzzing, not symbolic execution #define MAX_STR_LEN 6 +TEST(Runlength, BoringUnitTest) { + ASSERT_EQ(strcmp(encode(""), ""), 0); + ASSERT_EQ(strcmp(encode("a"), "aA"), 0); + ASSERT_EQ(strcmp(encode("aaabbbbbc"), "aCbEcA"), 0); +} + TEST(Runlength, EncodeDecode) { char* original = DeepState_CStrUpToLen(MAX_STR_LEN, "abcdef0123456789"); char* encoded = encode(original); From a30234fc7ca8030092ef467211e3d82603fb5ff7 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 13:00:31 -0700 Subject: [PATCH 07/10] fix log level arg --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff775b1..994854c 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ DeepState consists of a static library, used to write test harnesses, and comman ## Log Levels By default, DeepState is not very verbose about testing activity, -other than failing tests. The `--log-level` argument lowers the +other than failing tests. The `--log_level` argument lowers the threshold for output, with 0 = `DEBUG`, 1 = `TRACE` (output from the tests, including `printf`s), and 2 = INFO (DeepState messages, the default), 3 = `WARNING`, 4 = `ERROR`, and 5 = `EXTERNAL` (output from other programs such as From cbdda8bdea8a3e98ca8304ee70463a84f2b29431 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 13:02:34 -0700 Subject: [PATCH 08/10] reorg of README --- README.md | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 994854c..b778322 100644 --- a/README.md +++ b/README.md @@ -115,24 +115,6 @@ argument to see all DeepState options. DeepState consists of a static library, used to write test harnesses, and command-line _executors_ written in Python. At this time, the best documentation is in the [examples](/examples) and in our [paper](https://agroce.github.io/bar18.pdf). A more extensive example, using DeepState and libFuzzer to test a user-mode file system, is available [here](https://github.com/agroce/testfs); in particular the [Tests.cpp](https://github.com/agroce/testfs/blob/master/Tests.cpp) file and CMakeLists.txt show DeepState usage. -## Log Levels - -By default, DeepState is not very verbose about testing activity, -other than failing tests. The `--log_level` argument lowers the -threshold for output, with 0 = `DEBUG`, 1 = `TRACE` (output from the -tests, including `printf`s), and 2 = INFO (DeepState messages, the default), 3 = `WARNING`, -4 = `ERROR`, and 5 = `EXTERNAL` (output from other programs such as -libFuzzer), and 6 = `CRITICAL`/`FATAL` messages. - -## A Note on Mac OS and Forking - -Normally, when running a test for replay or fuzzing, DeepState forks -in order to cleanly handle crashes of a test. Unfortunately, `fork()` -on mac OS is extremely slow. When using the built-in fuzzer or -replaying tests, it is highly recommended to add the `--no_fork` -option on mac OS, unless you need the added crash handling (that is, -things aren't working without that option). - ## Example Code ``` @@ -248,6 +230,18 @@ CRITICAL: /Users/alex/deepstate/examples/Runlen.cpp(60): ORIGINAL: '91c499', ENC ERROR: Failed: Runlength_EncodeDecode ``` +## Log Levels + +By default, DeepState is not very verbose about testing activity, +other than failing tests. The `--log_level` argument lowers the +threshold for output, with 0 = `DEBUG`, 1 = `TRACE` (output from the +tests, including `printf`s), and 2 = INFO (DeepState messages, the default), 3 = `WARNING`, +4 = `ERROR`, and 5 = `EXTERNAL` (output from other programs such as +libFuzzer), and 6 = `CRITICAL`/`FATAL` messages. This can be very +useful when understanding what a DeepState harness is actually doing; +usually, setting `--log_level 1` in either fuzzing or symbolic +execution will give sufficient information to debug your test harness. + ## Built-In Fuzzer Every DeepState executable provides a simple built-in fuzzer that @@ -263,6 +257,15 @@ Note that while symbolic execution only works on Linux, without a fairly complex cross-compliation process, the brute force fuzzer works on macOS or (as far as we know) any Unix-like system. +## A Note on Mac OS and Forking + +Normally, when running a test for replay or fuzzing, DeepState forks +in order to cleanly handle crashes of a test. Unfortunately, `fork()` +on mac OS is extremely slow. When using the built-in fuzzer or +replaying tests, it is highly recommended to add the `--no_fork` +option on mac OS, unless you need the added crash handling (that is, +things aren't working without that option). + ## Fuzzing with libFuzzer If you install clang 6.0 or later, and run `cmake` when you install From 58df6801a146ce9fe833fd48576ce9de7bca1db5 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 13:13:57 -0700 Subject: [PATCH 09/10] more readme edits, typo fixes, etc. --- README.md | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b778322..2246f9b 100644 --- a/README.md +++ b/README.md @@ -184,30 +184,40 @@ TEST(Runlength, EncodeDecode) { ``` The code above (which can be found -[here](https://github.com/trailofbits/deepstate/blob/master/examples/Runlen.cpp) +[here](https://github.com/trailofbits/deepstate/blob/master/examples/Runlen.cpp)) shows an example of a DeepState test harness. Most of the code is just the functions to be tested. Using DeepState to test them requires: -- including the DeepState C++ header and using the DeepState namespace +- Including the DeepState C++ header and using the DeepState namespace -- defining at least one TEST, with names +- Defining at least one TEST, with names -- calling some DeepState APIs that produce data - - in this example, we see the `DeepState_CStrUpToLen` call tells +- Calling some DeepState APIs that produce data + - In this example, we see the `DeepState_CStrUpToLen` call tells DeepState to produce a string that has up the `MAX_STR_LEN` characters, chosen from those present in hex strings. -- optionally making some assertions about the correctness of the +- Optionally making some assertions about the correctness of the results - - in this example, the `ASSERT_LE` and `ASSERT_EQ` checks - - in the absence of any properties to check, DeepState can still + - In `Runlen.cpp` this is the `ASSERT_LE` and `ASSERT_EQ` checks. + - In the absence of any properties to check, DeepState can still look for memory safety violations, crashes, and other general - categories of undesireable behavior, like any fuzzer + categories of undesireable behavior, like any fuzzer. DeepState will also run the "BoringUnitTest," but it (like a traditional hand-written unit test) is simply a test of fixed inputs devised by a programmer. These inputs do not expose the bug in -`encode`. Using DeepState, however, it is easy to find the bug. Just +`encode`. Nor do the default values for the DeepState test: + +``` +~/deepstate/build/examples$ ./Runlen +TRACE: Running: Runlength_EncodeDecode from /Users/alex/deepstate/examples/Runlen.cpp(55) +TRACE: Passed: Runlength_EncodeDecode +TRACE: Running: Runlength_BoringUnitTest from /Users/alex/deepstate/examples/Runlen.cpp(49) +TRACE: Passed: Runlength_BoringUnitTest +``` + +Using DeepState, however, it is easy to find the bug. Just go into the `$DEEPSTATE/build/examples` directory and try: ```shell @@ -248,10 +258,13 @@ Every DeepState executable provides a simple built-in fuzzer that generates tests using completely random data. Using this fuzzer is as simple as calling the native executable with the `--fuzz` argument. The fuzzer also takes a `seed` and `timeout` (default of two minutes) -to control the fuzzing. If you want to actually save the test cases -generated, you need to add a `--output_test_dir` arument to tell -DeepState where to put the generated tests. By default fuzzing saves -only failing and crashing tests and only when given an output directory. +to control the fuzzing. By default fuzzing saves +only failing and crashing tests, and these only when given an output +directory. If you want to actually save the test cases +generated, you need to add a `--output_test_dir` argument to tell +DeepState where to put the generated tests, and if you want the +(totally random and unlikely to be high-quality) passing tests, you +need to add `--fuzz_save_passing`. Note that while symbolic execution only works on Linux, without a fairly complex cross-compliation process, the brute force fuzzer works @@ -261,10 +274,10 @@ on macOS or (as far as we know) any Unix-like system. Normally, when running a test for replay or fuzzing, DeepState forks in order to cleanly handle crashes of a test. Unfortunately, `fork()` -on mac OS is extremely slow. When using the built-in fuzzer or -replaying tests, it is highly recommended to add the `--no_fork` +on mac OS is _extremely_ slow. When using the built-in fuzzer or +replaying more than a few tests, it is highly recommended to add the `--no_fork` option on mac OS, unless you need the added crash handling (that is, -things aren't working without that option). +only when things aren't working without that option). ## Fuzzing with libFuzzer From 0ef1e9b4c2882f7419000cf01df4c0b9113748e1 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 4 Jan 2019 13:17:44 -0700 Subject: [PATCH 10/10] Multiple spelling error and readability fixes --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2246f9b..1096e81 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ results - In `Runlen.cpp` this is the `ASSERT_LE` and `ASSERT_EQ` checks. - In the absence of any properties to check, DeepState can still look for memory safety violations, crashes, and other general - categories of undesireable behavior, like any fuzzer. + categories of undesirable behavior, like any fuzzer. DeepState will also run the "BoringUnitTest," but it (like a traditional hand-written unit test) is simply a test of fixed inputs @@ -245,11 +245,11 @@ ERROR: Failed: Runlength_EncodeDecode By default, DeepState is not very verbose about testing activity, other than failing tests. The `--log_level` argument lowers the threshold for output, with 0 = `DEBUG`, 1 = `TRACE` (output from the -tests, including `printf`s), and 2 = INFO (DeepState messages, the default), 3 = `WARNING`, -4 = `ERROR`, and 5 = `EXTERNAL` (output from other programs such as -libFuzzer), and 6 = `CRITICAL`/`FATAL` messages. This can be very -useful when understanding what a DeepState harness is actually doing; -usually, setting `--log_level 1` in either fuzzing or symbolic +tests, including `printf`), 2 = `INFO` (DeepState messages; this is the default), 3 = `WARNING`, +4 = `ERROR`, 5 = `EXTERNAL` (output from other programs such as +libFuzzer), and 6 = `CRITICAL`/`FATAL` messages. Lowering the `log_level` can be very +useful for understanding what a DeepState harness is actually doing; +often, setting `--log_level 1` in either fuzzing or symbolic execution will give sufficient information to debug your test harness. ## Built-In Fuzzer @@ -267,7 +267,7 @@ DeepState where to put the generated tests, and if you want the need to add `--fuzz_save_passing`. Note that while symbolic execution only works on Linux, without a -fairly complex cross-compliation process, the brute force fuzzer works +fairly complex cross-compilation process, the brute force fuzzer works on macOS or (as far as we know) any Unix-like system. ## A Note on Mac OS and Forking