From ad544c99d75c687747a0531f36fd0b959631735d Mon Sep 17 00:00:00 2001 From: agroce Date: Mon, 17 Jun 2019 10:17:33 -0700 Subject: [PATCH 1/4] change log_level to min_log_level --- README.md | 6 +++--- bin/deepstate/common.py | 6 +++--- src/include/deepstate/DeepState.h | 10 +++++----- src/lib/DeepState.c | 8 ++++---- src/lib/Log.c | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1a9ca1e..f36d52b 100644 --- a/README.md +++ b/README.md @@ -293,13 +293,13 @@ ERROR: Failed: Runlength_EncodeDecode ## Log Levels By default, DeepState is not very verbose about testing activity, -other than failing tests. The `--log_level` argument lowers the +other than failing tests. The `--min_log_level` argument lowers the threshold for output, with 0 = `DEBUG`, 1 = `TRACE` (output from the tests, including from `printf`), 2 = INFO (DeepState messages, the default), 3 = `WARNING`, 4 = `ERROR`, 5 = `EXTERNAL` (output from other programs such as -libFuzzer), and 6 = `CRITICAL` messages. Lowering the `log_level` can be very +libFuzzer), and 6 = `CRITICAL` messages. Lowering the `min_log_level` can be very useful for understanding what a DeepState harness is actually doing; -often, setting `--log_level 1` in either fuzzing or symbolic +often, setting `--min_log_level 1` in either fuzzing or symbolic execution will give sufficient information to debug your test harness. diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index 18a6c71..1bf7e64 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -144,8 +144,8 @@ class DeepState(object): help="Verbosity level for symbolic execution tool (default: 1, lower means less output).") parser.add_argument( - "--log_level", default=2, type=int, - help="DeepState log level (default: 2), 0-6 (debug, trace, info, warning, error, external, critical).") + "--min_log_level", default=2, type=int, + help="Minimum DeepState log level to print (default: 2), 0-6 (debug, trace, info, warning, error, external, critical).") parser.add_argument( "binary", type=str, help="Path to the test binary to run.") @@ -273,7 +273,7 @@ class DeepState(object): LOG_LEVEL_ERROR: logging.ERROR, LOG_LEVEL_FATAL: logging.CRITICAL } - LOGGER.setLevel(logging_levels[args.log_level]) + LOGGER.setLevel(logging_levels[args.min_log_level]) if args.output_test_dir is not None: test_dir = os.path.join(args.output_test_dir, diff --git a/src/include/deepstate/DeepState.h b/src/include/deepstate/DeepState.h index fe81a0c..7edb22c 100644 --- a/src/include/deepstate/DeepState.h +++ b/src/include/deepstate/DeepState.h @@ -74,7 +74,7 @@ DECLARE_bool(fuzz); DECLARE_bool(fuzz_save_passing); DECLARE_bool(fork); -DECLARE_int(log_level); +DECLARE_int(min_log_level); DECLARE_int(seed); DECLARE_int(timeout); @@ -794,8 +794,8 @@ static int DeepState_RunSingleSavedTestDir(void) { int num_failed_tests = 0; struct DeepState_TestInfo *test = NULL; - if (!HAS_FLAG_log_level) { - FLAGS_log_level = 2; + if (!HAS_FLAG_min_log_level) { + FLAGS_min_log_level = 2; } DeepState_Setup(); @@ -876,8 +876,8 @@ static int DeepState_RunSavedTestCases(void) { int num_failed_tests = 0; struct DeepState_TestInfo *test = NULL; - if (!HAS_FLAG_log_level) { - FLAGS_log_level = 2; + if (!HAS_FLAG_min_log_level) { + FLAGS_min_log_level = 2; } DeepState_Setup(); diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 30ceb21..054b79d 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -47,7 +47,7 @@ DEFINE_bool(fuzz, false, "Perform brute force unguided fuzzing."); DEFINE_bool(fuzz_save_passing, false, "Save passing tests during fuzzing."); DEFINE_bool(fork, true, "Fork when running a test."); -DEFINE_int(log_level, 0, "Minimum level of logging to output."); +DEFINE_int(min_log_level, 0, "Minimum level of logging to output (default 2, 0=debug, 1=trace, 2=info, ...)."); DEFINE_int(seed, 0, "Seed for brute force fuzzing (uses time if not set)."); DEFINE_int(timeout, 120, "Timeout for brute force fuzzing."); @@ -739,8 +739,8 @@ bool DeepState_CatchAbandoned(void) { int DeepState_Fuzz(void){ DeepState_LogFormat(DeepState_LogInfo, "Starting fuzzing"); - if (!HAS_FLAG_log_level) { - FLAGS_log_level = 2; + if (!HAS_FLAG_min_log_level) { + FLAGS_min_log_level = 2; } if (HAS_FLAG_seed) { @@ -853,7 +853,7 @@ extern int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { const char* loud = getenv("LIBFUZZER_LOUD"); if (loud != NULL) { - FLAGS_log_level = 0; + FLAGS_min_log_level = 0; DeepState_LibFuzzerLoud = 1; } diff --git a/src/lib/Log.c b/src/lib/Log.c index b6d7016..a2f3556 100644 --- a/src/lib/Log.c +++ b/src/lib/Log.c @@ -77,7 +77,7 @@ char DeepState_LogBuf[DeepState_LogBufSize + 1] = {}; DEEPSTATE_NOINLINE void DeepState_Log(enum DeepState_LogLevel level, const char *str) { if ((DeepState_UsingLibFuzzer && !DeepState_LibFuzzerLoud && (level < DeepState_LogExternal)) || - (level < FLAGS_log_level)) { + (level < FLAGS_min_log_level)) { return; } memset(DeepState_LogBuf, 0, DeepState_LogBufSize); From c4725e8a1491e6cd6cb81dc0ab351a5e46df14c2 Mon Sep 17 00:00:00 2001 From: agroce Date: Mon, 17 Jun 2019 10:34:40 -0700 Subject: [PATCH 2/4] fix log level control in tests --- tests/logrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logrun.py b/tests/logrun.py index cd5926f..553983e 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -10,7 +10,7 @@ def logrun(cmd, file, timeout): sys.stderr.flush() with open(file, 'w') as outf: # We need to set log_level so we see ALL messages, for testing - p = subprocess.Popen(cmd + ["--log_level", "0"], stdout=outf, stderr=outf) + p = subprocess.Popen(cmd + ["--min_log_level", "0"], stdout=outf, stderr=outf) start = time.time() oldContents = "" lastOutput = time.time() From 34ae4f624d9cdf452d6224cba8faedbc6ea846de Mon Sep 17 00:00:00 2001 From: agroce Date: Mon, 17 Jun 2019 11:14:26 -0700 Subject: [PATCH 3/4] readme tweak --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index f36d52b..bbe63ee 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,60 @@ CRITICAL: /Users/alex/deepstate/examples/Runlen.cpp(60): ORIGINAL: '91c499', ENC ERROR: Failed: Runlength_EncodeDecode ``` +If you're using the DeepState docker, it's easy to also try libFuzzer +and AFL on the Runlen example: + +```shell +mkdir libfuzzer_runlen +./Runlen_LF libfuzzer_runlen -max_total_time=30 +./Runlen --input_test_files_dir libfuzzer_runlen +``` + +And you'll see a number of failures, e.g.: +``` +WARNING: No test specified, defaulting to last test defined (Runlength_EncodeDecode) +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: '4af4aa', ENCODED: '4AaAfA4AaA', ROUNDTRIP: '4af4a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//9e266f6cb627ce3bb7d717a6e569ade6b3633f23 failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: 'aaaaaa', ENCODED: 'aA', ROUNDTRIP: 'a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//d8fc60ccdd8f555c1858b9f0820f263e3d2b58ec failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: '4aaa', ENCODED: '4AaA', ROUNDTRIP: '4a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//3177c75208f2d35399842196dc8093243d5a8243 failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: 'aaa', ENCODED: 'aA', ROUNDTRIP: 'a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//9842926af7ca0a8cca12604f945414f07b01e13d failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: 'aaa', ENCODED: 'aA', ROUNDTRIP: 'a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//85e53271e14006f0265921d02d4d736cdc580b0b failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: 'aaaaa', ENCODED: 'aA', ROUNDTRIP: 'a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//241cbd6dfb6e53c43c73b62f9384359091dcbf56 failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: 'aa', ENCODED: 'aA', ROUNDTRIP: 'a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//05a79f06cf3f67f726dae68d18a2290f6c9a50c9 failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: '25aaaa', ENCODED: '2A5AaA', ROUNDTRIP: '25a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//419c3b754bacd6fc14ff9a932c5e2089d6dfcab5 failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: 'aaaa', ENCODED: 'aA', ROUNDTRIP: 'a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//bb589d0621e5472f470fa3425a234c74b1e202e8 failed +CRITICAL: /home/user/deepstate/examples/Runlen.cpp(60): ORIGINAL: '97aa', ENCODED: '9A7AaA', ROUNDTRIP: '97a' +ERROR: Failed: Runlength_EncodeDecode +ERROR: Test case libfuzzer_runlen//ca61c43b0e3ff0a8eccf3136996c9f1d9bfd627c failed +INFO: Ran 16 tests; 10 tests failed +``` + +```shell +mkdir afl_seeds +echo "ok" >& seeds/seed +afl-fuzz -i seeds -o afl_runlen -- ./Runlen_AFL --input_test_file @@ --no_fork --abort_on_fail +``` + +You'll have to stop this with Ctrl-C. The `afl_runlen/crashes` +directory will contain crashing inputs AFL found. + ## Log Levels By default, DeepState is not very verbose about testing activity, From 57de6ccc664fca3eae4a090671da0a1375bca74d Mon Sep 17 00:00:00 2001 From: agroce Date: Mon, 17 Jun 2019 11:15:29 -0700 Subject: [PATCH 4/4] text for AFL --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bbe63ee..83682aa 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,8 @@ ERROR: Test case libfuzzer_runlen//ca61c43b0e3ff0a8eccf3136996c9f1d9bfd627c fail INFO: Ran 16 tests; 10 tests failed ``` +Using AFL is similarly easy: + ```shell mkdir afl_seeds echo "ok" >& seeds/seed