DeforaNetworks/deepstate
3
0
Fork 0
Code Releases Activity

Merge pull request #111 from trailofbits/verbose_input_reading

Browse Source 
Allow verbose output of structure of test and provide a simple test reducer
This commit is contained in:
Peter Goodman 2018-09-03 19:43:35 -04:00 committed by GitHub
commit 4fa87331c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 259 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
README.md
View File

@ -144,6 +144,57 @@ memory during a test, if that memory would not be freed on a test
failure. This will leak memory and libFuzzer will run out of memory
very quickly in each fuzzing session.
## Test case reduction
While tests generated by symbolic execution are likely to be highly
concise already, fuzzer-generated tests may be much larger than they
need to be.
DeepState provides a test case reducer to shrink tests intelligently,
aware of the structure of a DeepState test. For example, if your
executable is named `TestFileSystem` and the test you want to reduce
is named `rmdirfail.test` you would use it like this:
```shell
deepstate-reduce ./TestFileSystem rmdirfail.test minrmdirfail.test
```
In many cases, this will result in finding a different failure or
crash that allow smaller test cases, so you can also provide a string
that controls the criteria for which test outputs are considered valid
reductions (by default, the reducer looks for any test that fails or
crashes). Only outputs containing the `--criteria` are considered to
be valid reductions:
```shell
deepstate-reduce ./TestFileSystem rmdirfail.test minrmdirfail.test --criteria "FATAL: /root/testfs/super.c(252)"
```
The output will look something like:
```
ORIGINAL TEST HAS 119 BYTES
ONEOF REMOVAL REDUCED TEST TO 103 BYTES
ONEOF REMOVAL REDUCED TEST TO 87 BYTES
ONEOF REMOVAL REDUCED TEST TO 67 BYTES
ONEOF REMOVAL REDUCED TEST TO 51 BYTES
BYTE RANGE REMOVAL REDUCED TEST TO 50 BYTES
BYTE RANGE REMOVAL REDUCED TEST TO 49 BYTES
BYTE REDUCTION: BYTE 3 FROM 4 TO 0
BYTE REDUCTION: BYTE 43 FROM 4 TO 0
ONEOF REMOVAL REDUCED TEST TO 33 BYTES
ONEOF REMOVAL REDUCED TEST TO 17 BYTES
BYTE REDUCTION: BYTE 7 FROM 2 TO 1
BYTE REDUCTION: BYTE 15 FROM 2 TO 1
NO REDUCTIONS FOUND
PADDING TEST WITH 3 ZEROS
WRITING REDUCED TEST WITH 20 BYTES TO minrmdirfail.test
```
You can use `--which_test <testname>` to specify which test to
run, as with the `--input_which_test` options to test replay.
## Fuzzing with AFL
DeepState can also be used with a file-based fuzzer (e.g. AFL). There

184
bin/deepstate/reducer.py Normal file
View File

@ -0,0 +1,184 @@
#!/usr/bin/env python
# Copyright (c) 2018 Trail of Bits, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import argparse
import subprocess
def main():
parser = argparse.ArgumentParser(description="Intelligently reduce test case")
parser.add_argument(
"binary", type=str, help="Path to the test binary to run.")
parser.add_argument(
"input_test", type=str, help="Path to test to reduce.")
parser.add_argument(
"output_test", type=str, help="Path for reduced test.")
parser.add_argument(
"--which_test", type=str, help="Which test to run (equivalent to --input_which_test).", default=None)
parser.add_argument(
"--criteria", type=str, help="String to search for in valid reduction outputs.",
default=None)
parser.add_argument(
"--search", action="store_true", help="Allow initial test to not satisfy criteria (search for test).",
default=None)
args = parser.parse_args()
deepstate = args.binary
test = args.input_test
out = args.output_test
checkString = args.criteria
whichTest = args.which_test
def runCandidate(candidate):
with open(".reducer.out", 'w') as outf:
cmd = [deepstate + " --input_test_file " +
candidate + " --verbose_reads"]
if whichTest is not None:
cmd += ["--input_which_test", whichTest]
subprocess.call(cmd, shell=True, stdout=outf, stderr=outf)
result = []
with open(".reducer.out", 'r') as inf:
for line in inf:
result.append(line)
return result
def checks(result):
for line in result:
if checkString:
if checkString in line:
return True
else:
if "ERROR: Failed:" in line:
return True
if "ERROR: Crashed" in line:
return True
return False
def writeAndRunCandidate(test):
with open(".candidate.test", 'wb') as outf:
outf.write(test)
r = runCandidate(".candidate.test")
return r
def structure(result):
OneOfs = []
currentOneOf = []
for line in result:
if "STARTING OneOf CALL" in line:
currentOneOf.append(-1)
elif "Reading byte at" in line:
lastRead = int(line.split()[-1])
if currentOneOf[-1] == -1:
currentOneOf[-1] = lastRead
elif "FINISHED OneOf CALL" in line:
OneOfs.append((currentOneOf[-1], lastRead))
currentOneOf = currentOneOf[:-1]
return (OneOfs, lastRead)
initial = runCandidate(test)
if (not args.search) and (not checks(initial)):
print("STARTING TEST DOES NOT SATISFY REDUCTION CRITERIA")
return 1
with open(test, 'rb') as test:
currentTest = bytearray(test.read())
print("ORIGINAL TEST HAS", len(currentTest), "BYTES")
s = structure(initial)
if (s[1]+1) < len(currentTest):
print("LAST BYTE READ IS", s[1])
print("SHRINKING TO IGNORE UNREAD BYTES")
currentTest = currentTest[:s[1]+1]
changed = True
while changed:
changed = False
cuts = s[0]
for c in cuts:
newTest = currentTest[:c[0]] + currentTest[c[1]+1:]
r = writeAndRunCandidate(newTest)
if checks(r):
print("ONEOF REMOVAL REDUCED TEST TO", len(newTest), "BYTES")
changed = True
break
if not changed:
for b in range(0, len(currentTest)):
for v in range(b+1, len(currentTest)):
newTest = currentTest[:b] + currentTest[v:]
r = writeAndRunCandidate(newTest)
if checks(r):
print("BYTE RANGE REMOVAL REDUCED TEST TO", len(newTest), "BYTES")
changed = True
break
if changed:
break
if not changed:
for b in range(0, len(currentTest)):
for v in range(0, currentTest[b]):
newTest = bytearray(currentTest)
newTest[b] = v
r = writeAndRunCandidate(newTest)
if checks(r):
print("BYTE REDUCTION: BYTE", b, "FROM", currentTest[b], "TO", v)
changed = True
break
if changed:
break
if not changed:
for b in range(0, len(currentTest)):
if currentTest[b] == 0:
continue
newTest = bytearray(currentTest)
newTest[b] = currentTest[b]-1
newTest = newTest[:b+1] + newTest[b+2:]
r = writeAndRunCandidate(newTest)
if checks(r):
print("BYTE REDUCE AND DELETE AT BYTE", b)
changed = True
break
if changed:
currentTest = newTest
s = structure(r)
print("NO REDUCTIONS FOUND")
if (s[1] + 1) > len(currentTest):
print("PADDING TEST WITH", (s[1] + 1) - len(currentTest), "ZEROS")
padding = bytearray('\x00' * ((s[1] + 1) - len(currentTest)))
currentTest = currentTest + padding
print()
print("WRITING REDUCED TEST WITH", len(currentTest), "BYTES TO", out)
with open(out, 'wb') as outf:
outf.write(currentTest)
return 0
if "__main__" == __name__:
exit(main())

1
bin/setup.py.in
View File

@ -36,5 +36,6 @@ setuptools.setup(
'deepstate = deepstate.main_manticore:main',
'deepstate-angr = deepstate.main_angr:main',
'deepstate-manticore = deepstate.main_manticore:main',
'deepstate-reduce = deepstate.reducer:main',
]
})

1
src/include/deepstate/DeepState.h
View File

@ -62,6 +62,7 @@ DECLARE_string(output_test_dir);
DECLARE_bool(take_over);
DECLARE_bool(abort_on_fail);
DECLARE_bool(verbose_reads);
enum {
DeepState_InputSize = 8192

6
src/include/deepstate/DeepState.hpp
View File

@ -327,10 +327,16 @@ inline static void ForAll(Closure func) {
template <typename... FuncTys>
inline static void OneOf(FuncTys&&... funcs) {
if (FLAGS_verbose_reads) {
printf("STARTING OneOf CALL\n");
}
std::function<void(void)> func_arr[sizeof...(FuncTys)] = {funcs...};
unsigned index = DeepState_UIntInRange(
0U, static_cast<unsigned>(sizeof...(funcs))-1);
func_arr[Pump(index, sizeof...(funcs))]();
if (FLAGS_verbose_reads) {
printf("FINISHED OneOf CALL\n");
}
}
inline static char OneOf(const char *str) {

16
src/lib/DeepState.c
View File

@ -36,6 +36,7 @@ DEFINE_string(output_test_dir, "", "Directory where tests will be saved.");
DEFINE_bool(take_over, false, "Replay test cases in take-over mode.");
DEFINE_bool(abort_on_fail, false, "Abort on file replay failure (useful in file fuzzing).");
DEFINE_bool(verbose_reads, false, "Report on bytes being read during execution of test.");
/* Pointer to the last registers DeepState_TestInfo data structure */
struct DeepState_TestInfo *DeepState_LastTestInfo = NULL;
@ -140,6 +141,9 @@ void DeepState_SymbolizeData(void *begin, void *end) {
if (DeepState_InputIndex >= DeepState_InputSize) {
DeepState_Abandon("Read too many symbols");
}
if (FLAGS_verbose_reads) {
printf("Reading byte at %u\n", DeepState_InputIndex);
}
bytes[i] = DeepState_Input[DeepState_InputIndex++];
}
}
@ -216,6 +220,9 @@ int DeepState_Bool(void) {
if (DeepState_InputIndex >= DeepState_InputSize) {
DeepState_Abandon("Read too many symbols");
}
if (FLAGS_verbose_reads) {
printf("Reading byte as boolean at %u\n", DeepState_InputIndex);
}
return DeepState_Input[DeepState_InputIndex++] & 1;
}
@ -225,10 +232,19 @@ int DeepState_Bool(void) {
DeepState_Abandon("Read too many symbols"); \
} \
type val = 0; \
if (FLAGS_verbose_reads) { \
printf("STARTING MULTI-BYTE READ\n"); \
} \
_Pragma("unroll") \
for (size_t i = 0; i < sizeof(type); ++i) { \
if (FLAGS_verbose_reads) { \
printf("Reading byte at %u\n", DeepState_InputIndex); \
} \
val = (val << 8) | ((type) DeepState_Input[DeepState_InputIndex++]); \
} \
if (FLAGS_verbose_reads) { \
printf("FINISHED MULTI-BYTE READ\n"); \
} \
return val; \
}