DeforaNetworks/deepstate
3
0
Fork 0
Code Releases Activity

Merge pull request #97 from trailofbits/dev-python3.6

Browse Source 
Experimental python3.6 version
This commit is contained in:
Peter Goodman
2019-03-28 17:15:14 -04:00
committed by GitHub
parent 4dfd951882 151354cb90
commit 30211fba72
6 changed files with 64 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+20 -18
View File
@@ -1,14 +1,16 @@
language: bash
dist: xenial
sudo: true
#python:
#- "2.7"
language: python
python:
- 3.6.5
install:
- sudo apt-get -y update
- sudo apt-get -y install build-essential gcc-multilib cmake python python-pip python-setuptools libffi-dev python-nose
- pip2 install pyflakes --user
- pip2 install angr==7.8.9.26 --user
- pip2 install https://github.com/trailofbits/manticore/archive/last_python2.zip --user
- sudo apt-get -y install build-essential gcc-multilib cmake python3-pip python3-setuptools libffi-dev python3-nose
- pip3 install pip
- pip3.6 -V
- pip3.6 install pyflakes
- pip3.6 install angr
- pip3.6 install https://github.com/trailofbits/manticore/archive/master.zip
- mkdir build
- cd build
- cmake ..
@@ -29,15 +31,15 @@ env:
script:
- pyflakes bin/deepstate/*.py
- pyflakes tests/*.py
- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic.py ; fi
- if [ $TASK = CRASH ]; then nosetests tests/test_crash.py ; fi
- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture.py ; fi
- if [ $TASK = KLEE ]; then nosetests tests/test_klee.py ; fi
- if [ $TASK = LISTS ]; then nosetests tests/test_lists.py ; fi
- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof.py ; fi
- if [ $TASK = RUNLEN ]; then nosetests tests/test_runlen.py ; fi
- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow.py ; fi
- if [ $TASK = PRIMES ]; then nosetests tests/test_primes.py ; fi
- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting.py ; fi
- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover.py ; fi
- if [ $TASK = ARITHMETIC ]; then nosetests3 tests/test_arithmetic.py ; fi
- if [ $TASK = CRASH ]; then nosetests3 tests/test_crash.py ; fi
- if [ $TASK = FIXTURE ]; then nosetests3 tests/test_fixture.py ; fi
- if [ $TASK = KLEE ]; then nosetests3 tests/test_klee.py ; fi
- if [ $TASK = LISTS ]; then nosetests3 tests/test_lists.py ; fi
- if [ $TASK = ONEOF ]; then nosetests3 tests/test_oneof.py ; fi
- if [ $TASK = RUNLEN ]; then nosetests3 tests/test_runlen.py ; fi
- if [ $TASK = OVERFLOW ]; then nosetests3 tests/test_overflow.py ; fi
- if [ $TASK = PRIMES ]; then nosetests3 tests/test_primes.py ; fi
- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests3 tests/test_streamingandformatting.py ; fi
- if [ $TASK = TAKEOVER ]; then nosetests3 tests/test_takeover.py ; fi
CMakeLists.txt
+1 -1
View File
@@ -56,7 +56,7 @@ set(CMAKE_C_FLAGS_RELEASE "-O3")
set(CMAKE_CXX_FLAGS_DEBUG "-O3")
set(CMAKE_CXX_FLAGS_RELEASE "-O3")
find_program(PYTHON "python2.7")
find_program(PYTHON "python3.6")
# Enable the GNU extensions
set(CMAKE_CXX_EXTENSIONS ON)
bin/deepstate/common.py
+11 -8
View File
@@ -17,8 +17,9 @@ logging.basicConfig()
logging.addLevelName(15, "TRACE")
import argparse
#import md5
import hashlib
import functools
import md5
import os
import struct
@@ -160,7 +161,7 @@ class DeepState(object):
def read_c_string(self, ea, concretize=True, constrain=False):
"""Read a NUL-terminated string from `ea`."""
assert isinstance(ea, (int, long))
assert isinstance(ea, (int))
chars = []
while True:
b, ea = self.read_uint8_t(ea, concretize=concretize, constrain=constrain)
@@ -254,7 +255,7 @@ class DeepState(object):
# Create the symbols that feed API functions like `DeepState_Int`.
symbols = []
for i, ea in enumerate(xrange(apis['InputBegin'], apis['InputEnd'])):
for i, ea in enumerate(range(apis['InputBegin'], apis['InputEnd'])):
symbol = self.create_symbol('DEEP_INPUT_{}'.format(i), 8)
self.write_uint8_t(ea, symbol)
symbols.append(symbol)
@@ -309,7 +310,7 @@ class DeepState(object):
for b in byte_str:
if isinstance(b, str):
new_bytes.extend(ord(bn) for bn in b)
elif isinstance(b, (int, long)):
elif isinstance(b, (int)):
new_bytes.append(b)
elif isinstance(b, (list, tuple)):
new_bytes.extend(self._concretize_bytes(b))
@@ -357,7 +358,9 @@ class DeepState(object):
return
test_dir = self.context['test_dir']
test_name = md5.new(input_bytes).hexdigest()
md5 = hashlib.md5()
md5.update(input_bytes)
test_name = md5.hexdigest()
passing = False
if self.context['failed']:
@@ -397,7 +400,7 @@ class DeepState(object):
# likely to get the same concrete byte values across different tools (e.g.
# Manticore, Angr).
input_bytes = bytearray()
for i in xrange(input_length):
for i in range(input_length):
b = self.concretize_min(symbols[i], constrain=True)
input_bytes.append(b)
@@ -493,7 +496,7 @@ class DeepState(object):
begin_ea, end_ea))
self.abandon_test()
for i in xrange(end_ea - begin_ea):
for i in range(end_ea - begin_ea):
val, _ = self.read_uint8_t(begin_ea + i, concretize=True, constrain=True)
self.write_uint8_t(begin_ea + i, val)
@@ -580,7 +583,7 @@ class DeepState(object):
format_str = self.read_c_string(format_ea)[0]
unpack_str = self.read_c_string(unpack_ea)[0]
uint64_bytes = []
for i in xrange(8):
for i in range(8):
b, _ = self.read_uint8_t(uint64_ea + i, concretize=False)
uint64_bytes.append(b)
bin/deepstate/main_angr.py
+8 -8
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3.6
# Copyright (c) 2017 Trail of Bits, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -41,12 +41,12 @@ class DeepAngr(DeepState):
return self.state.globals
def is_symbolic(self, val):
if isinstance(val, (int, long)):
if isinstance(val, (int, int)):
return False
return self.state.se.symbolic(val)
return self.state.solver.symbolic(val)
def create_symbol(self, name, size_in_bits):
return self.state.se.Unconstrained('name', size_in_bits)
return self.state.solver.Unconstrained('name', size_in_bits)
def read_uintptr_t(self, ea, concretize=True, constrain=False):
addr_size_bytes = self.state.arch.bits // 8
@@ -89,7 +89,7 @@ class DeepAngr(DeepState):
return ea + 4
def concretize(self, val, constrain=False):
if isinstance(val, (int, long)):
if isinstance(val, (int, int)):
return val
elif isinstance(val, str):
assert len(val) == 1
@@ -102,7 +102,7 @@ class DeepAngr(DeepState):
return concrete_val
def concretize_min(self, val, constrain=False):
if isinstance(val, (int, long)):
if isinstance(val, (int, int)):
return val
concrete_val = self.state.solver.min(val)
if constrain:
@@ -110,7 +110,7 @@ class DeepAngr(DeepState):
return concrete_val
def concretize_max(self, val, constrain=False):
if isinstance(val, (int, long)):
if isinstance(val, (int, int)):
return val
concrete_val = self.state.solver.max(val)
if constrain:
@@ -119,7 +119,7 @@ class DeepAngr(DeepState):
def concretize_many(self, val, max_num):
assert 0 < max_num
if isinstance(val, (int, long)):
if isinstance(val, (int, int)):
return [val]
return self.state.solver.eval_upto(val, max_num, cast_to=int)
bin/deepstate/main_manticore.py
+23 -21
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3.6
# Copyright (c) 2017 Trail of Bits, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,14 +18,14 @@ logging.basicConfig()
import sys
try:
import manticore
import manticore
import manticore.native
except Exception as e:
if "Z3NotFoundError" in repr(type(e)):
print "Manticore requires Z3 to be installed."
print("Manticore requires Z3 to be installed.")
sys.exit(255)
else:
raise
import multiprocessing
import traceback
from .common import DeepState, TestInfo
@@ -52,7 +52,7 @@ class DeepManticore(DeepState):
return manticore.issymbolic(val)
def create_symbol(self, name, size_in_bits):
return self.state.new_symbolic_value(size_in_bits, name)
return self.state.new_symbolic_value(size_in_bits)
def read_uintptr_t(self, ea, concretize=True, constrain=False):
addr_size_bits = self.state.cpu.address_bit_size
@@ -92,7 +92,7 @@ class DeepManticore(DeepState):
return ea + 4
def concretize(self, val, constrain=False):
if isinstance(val, (int, long)):
if isinstance(val, (int)):
return val
elif isinstance(val, str):
assert len(val) == 1
@@ -108,7 +108,7 @@ class DeepManticore(DeepState):
return concrete_val
def concretize_min(self, val, constrain=False):
if isinstance(val, (int, long)):
if isinstance(val, (int)):
return val
concrete_val = min(self.state.concretize(val, policy='MINMAX'))
if constrain:
@@ -116,7 +116,7 @@ class DeepManticore(DeepState):
return concrete_val
def concretize_max(self, val, constrain=False):
if isinstance(val, (int, long)):
if isinstance(val, (int)):
return val
concrete_val = max(self.state.concretize(val, policy='MINMAX'))
if constrain:
@@ -125,7 +125,7 @@ class DeepManticore(DeepState):
def concretize_many(self, val, max_num):
assert 0 < max_num
if isinstance(val, (int, long)):
if isinstance(val, (int)):
return [val]
return self.state.solve_n(val, max_num)
@@ -266,7 +266,7 @@ def _is_program_crash(reason):
if not isinstance(reason, TerminateState):
return False
return 'Invalid memory access' in reason.message
return 'Invalid memory access' in str(reason)
def _is_program_exit(reason):
@@ -277,7 +277,7 @@ def _is_program_exit(reason):
if not isinstance(reason, TerminateState):
return False
return 'Program finished with exit status' in reason.message
return 'Program finished with exit status' in str(reason)
def done_test(_, state, state_id, reason):
@@ -289,7 +289,8 @@ def done_test(_, state, state_id, reason):
# DeepState API, so we can just report it as is. Otherwise, we check to see if
# it was due to behavior that would typically crash the program being analyzed.
# If so, we save it as a crash. If not, we abandon it.
if OUR_TERMINATION_REASON not in reason:
if str(OUR_TERMINATION_REASON) != str(reason):
if _is_program_crash(reason):
L.info("State {} terminated due to crashing program behavior: {}".format(
state_id, reason))
@@ -330,7 +331,8 @@ def find_symbol_ea(m, name):
def do_run_test(state, apis, test, hook_test=False):
"""Run an individual test case."""
state.cpu.PC = test.ea
m = manticore.Manticore(state, sys.argv[1:])
m = manticore.native.Manticore(state, sys.argv[1:])
#m = MainThreadWrapper(m, _CONTROLLER)
m.verbosity(1)
state = m.initial_state
@@ -364,7 +366,7 @@ def do_run_test(state, apis, test, hook_test=False):
m.add_hook(test.ea, hook(hook_TakeOver))
m.subscribe('will_terminate_state', done_test)
m.run()
m.run(procs=1)
def run_test(state, apis, test, hook_test=False):
@@ -377,7 +379,7 @@ def run_test(state, apis, test, hook_test=False):
def run_tests(args, state, apis):
"""Run all of the test cases."""
pool = multiprocessing.Pool(processes=max(1, args.num_workers))
#pool = multiprocessing.Pool(processes=max(1, args.num_workers))
results = []
mc = DeepManticore(state)
tests = mc.find_test_cases()
@@ -386,11 +388,11 @@ def run_tests(args, state, apis):
len(tests), args.num_workers))
for test in tests:
res = pool.apply_async(run_test, (state, apis, test))
res = run_test(state, apis, test)
results.append(res)
pool.close()
pool.join()
#pool.close()
#pool.join()
exit(0)
@@ -435,7 +437,7 @@ def main_takeover(m, args, takeover_symbol):
takeover_hook = lambda state: run_test(state, apis, fake_test, hook_test)
m.add_hook(takeover_ea, takeover_hook)
m.run()
m.run(procs=1)
def main_unit_test(m, args):
@@ -459,14 +461,14 @@ def main_unit_test(m, args):
del mc
m.add_hook(setup_ea, lambda state: run_tests(args, state, apis))
m.run()
m.run(procs=1)
def main():
args = DeepManticore.parse_args()
try:
m = manticore.Manticore(args.binary)
m = manticore.native.Manticore(args.binary)
except Exception as e:
L.critical("Cannot create Manticore instance on binary {}: {}".format(
args.binary, e))
bin/setup.py.in
+1 -4
View File
@@ -30,10 +30,7 @@ setuptools.setup(
author_email="peter@trailofbits.com",
license="Apache-2.0",
keywords="tdd testing symbolic execution",
install_requires=[
"claripy==7.8.6.16 ; sys_platform != 'darwin'",
"angr==7.8.7.1 ; sys_platform != 'darwin'",
"manticore==0.1.10 ; sys_platform != 'darwin'"],
install_requires=[], #'claripy==7.8.6.16','angr==7.8.7.1', 'manticore'],
entry_points={
'console_scripts': [
'deepstate = deepstate.main_manticore:main',