diff --git a/.travis.yml b/.travis.yml index fb8741a..bfed9b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,16 @@ -language: bash dist: xenial sudo: true -#python: -#- "2.7" +language: python +python: + - 3.6.5 install: - sudo apt-get -y update -- sudo apt-get -y install build-essential gcc-multilib cmake python python-pip python-setuptools libffi-dev python-nose -- pip2 install pyflakes --user -- pip2 install angr==7.8.9.26 --user -- pip2 install https://github.com/trailofbits/manticore/archive/last_python2.zip --user +- sudo apt-get -y install build-essential gcc-multilib cmake python3-pip python3-setuptools libffi-dev python3-nose +- pip3 install pip +- pip3.6 -V +- pip3.6 install pyflakes +- pip3.6 install angr +- pip3.6 install https://github.com/trailofbits/manticore/archive/master.zip - mkdir build - cd build - cmake .. @@ -29,15 +31,15 @@ env: script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py -- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic.py ; fi -- if [ $TASK = CRASH ]; then nosetests tests/test_crash.py ; fi -- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture.py ; fi -- if [ $TASK = KLEE ]; then nosetests tests/test_klee.py ; fi -- if [ $TASK = LISTS ]; then nosetests tests/test_lists.py ; fi -- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof.py ; fi -- if [ $TASK = RUNLEN ]; then nosetests tests/test_runlen.py ; fi -- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow.py ; fi -- if [ $TASK = PRIMES ]; then nosetests tests/test_primes.py ; fi -- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting.py ; fi -- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover.py ; fi +- if [ $TASK = ARITHMETIC ]; then nosetests3 tests/test_arithmetic.py ; fi +- if [ $TASK = CRASH ]; then nosetests3 tests/test_crash.py ; fi +- if [ $TASK = FIXTURE ]; then nosetests3 tests/test_fixture.py ; fi +- if [ $TASK = KLEE ]; then nosetests3 tests/test_klee.py ; fi +- if [ $TASK = LISTS ]; then nosetests3 tests/test_lists.py ; fi +- if [ $TASK = ONEOF ]; then nosetests3 tests/test_oneof.py ; fi +- if [ $TASK = RUNLEN ]; then nosetests3 tests/test_runlen.py ; fi +- if [ $TASK = OVERFLOW ]; then nosetests3 tests/test_overflow.py ; fi +- if [ $TASK = PRIMES ]; then nosetests3 tests/test_primes.py ; fi +- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests3 tests/test_streamingandformatting.py ; fi +- if [ $TASK = TAKEOVER ]; then nosetests3 tests/test_takeover.py ; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 9652957..489751c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,7 @@ set(CMAKE_C_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_DEBUG "-O3") set(CMAKE_CXX_FLAGS_RELEASE "-O3") -find_program(PYTHON "python2.7") +find_program(PYTHON "python3.6") # Enable the GNU extensions set(CMAKE_CXX_EXTENSIONS ON) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index 0004514..8994f09 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -17,8 +17,9 @@ logging.basicConfig() logging.addLevelName(15, "TRACE") import argparse +#import md5 +import hashlib import functools -import md5 import os import struct @@ -160,7 +161,7 @@ class DeepState(object): def read_c_string(self, ea, concretize=True, constrain=False): """Read a NUL-terminated string from `ea`.""" - assert isinstance(ea, (int, long)) + assert isinstance(ea, (int)) chars = [] while True: b, ea = self.read_uint8_t(ea, concretize=concretize, constrain=constrain) @@ -254,7 +255,7 @@ class DeepState(object): # Create the symbols that feed API functions like `DeepState_Int`. symbols = [] - for i, ea in enumerate(xrange(apis['InputBegin'], apis['InputEnd'])): + for i, ea in enumerate(range(apis['InputBegin'], apis['InputEnd'])): symbol = self.create_symbol('DEEP_INPUT_{}'.format(i), 8) self.write_uint8_t(ea, symbol) symbols.append(symbol) @@ -309,7 +310,7 @@ class DeepState(object): for b in byte_str: if isinstance(b, str): new_bytes.extend(ord(bn) for bn in b) - elif isinstance(b, (int, long)): + elif isinstance(b, (int)): new_bytes.append(b) elif isinstance(b, (list, tuple)): new_bytes.extend(self._concretize_bytes(b)) @@ -357,7 +358,9 @@ class DeepState(object): return test_dir = self.context['test_dir'] - test_name = md5.new(input_bytes).hexdigest() + md5 = hashlib.md5() + md5.update(input_bytes) + test_name = md5.hexdigest() passing = False if self.context['failed']: @@ -397,7 +400,7 @@ class DeepState(object): # likely to get the same concrete byte values across different tools (e.g. # Manticore, Angr). input_bytes = bytearray() - for i in xrange(input_length): + for i in range(input_length): b = self.concretize_min(symbols[i], constrain=True) input_bytes.append(b) @@ -493,7 +496,7 @@ class DeepState(object): begin_ea, end_ea)) self.abandon_test() - for i in xrange(end_ea - begin_ea): + for i in range(end_ea - begin_ea): val, _ = self.read_uint8_t(begin_ea + i, concretize=True, constrain=True) self.write_uint8_t(begin_ea + i, val) @@ -580,7 +583,7 @@ class DeepState(object): format_str = self.read_c_string(format_ea)[0] unpack_str = self.read_c_string(unpack_ea)[0] uint64_bytes = [] - for i in xrange(8): + for i in range(8): b, _ = self.read_uint8_t(uint64_ea + i, concretize=False) uint64_bytes.append(b) diff --git a/bin/deepstate/main_angr.py b/bin/deepstate/main_angr.py index a35bd58..94e53c0 100644 --- a/bin/deepstate/main_angr.py +++ b/bin/deepstate/main_angr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3.6 # Copyright (c) 2017 Trail of Bits, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,12 +41,12 @@ class DeepAngr(DeepState): return self.state.globals def is_symbolic(self, val): - if isinstance(val, (int, long)): + if isinstance(val, (int, int)): return False - return self.state.se.symbolic(val) + return self.state.solver.symbolic(val) def create_symbol(self, name, size_in_bits): - return self.state.se.Unconstrained('name', size_in_bits) + return self.state.solver.Unconstrained('name', size_in_bits) def read_uintptr_t(self, ea, concretize=True, constrain=False): addr_size_bytes = self.state.arch.bits // 8 @@ -89,7 +89,7 @@ class DeepAngr(DeepState): return ea + 4 def concretize(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int, int)): return val elif isinstance(val, str): assert len(val) == 1 @@ -102,7 +102,7 @@ class DeepAngr(DeepState): return concrete_val def concretize_min(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int, int)): return val concrete_val = self.state.solver.min(val) if constrain: @@ -110,7 +110,7 @@ class DeepAngr(DeepState): return concrete_val def concretize_max(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int, int)): return val concrete_val = self.state.solver.max(val) if constrain: @@ -119,7 +119,7 @@ class DeepAngr(DeepState): def concretize_many(self, val, max_num): assert 0 < max_num - if isinstance(val, (int, long)): + if isinstance(val, (int, int)): return [val] return self.state.solver.eval_upto(val, max_num, cast_to=int) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index 26d6672..dd591a5 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3.6 # Copyright (c) 2017 Trail of Bits, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,14 @@ logging.basicConfig() import sys try: - import manticore + import manticore + import manticore.native except Exception as e: if "Z3NotFoundError" in repr(type(e)): - print "Manticore requires Z3 to be installed." + print("Manticore requires Z3 to be installed.") sys.exit(255) else: raise -import multiprocessing import traceback from .common import DeepState, TestInfo @@ -52,7 +52,7 @@ class DeepManticore(DeepState): return manticore.issymbolic(val) def create_symbol(self, name, size_in_bits): - return self.state.new_symbolic_value(size_in_bits, name) + return self.state.new_symbolic_value(size_in_bits) def read_uintptr_t(self, ea, concretize=True, constrain=False): addr_size_bits = self.state.cpu.address_bit_size @@ -92,7 +92,7 @@ class DeepManticore(DeepState): return ea + 4 def concretize(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int)): return val elif isinstance(val, str): assert len(val) == 1 @@ -108,7 +108,7 @@ class DeepManticore(DeepState): return concrete_val def concretize_min(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int)): return val concrete_val = min(self.state.concretize(val, policy='MINMAX')) if constrain: @@ -116,7 +116,7 @@ class DeepManticore(DeepState): return concrete_val def concretize_max(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int)): return val concrete_val = max(self.state.concretize(val, policy='MINMAX')) if constrain: @@ -125,7 +125,7 @@ class DeepManticore(DeepState): def concretize_many(self, val, max_num): assert 0 < max_num - if isinstance(val, (int, long)): + if isinstance(val, (int)): return [val] return self.state.solve_n(val, max_num) @@ -266,7 +266,7 @@ def _is_program_crash(reason): if not isinstance(reason, TerminateState): return False - return 'Invalid memory access' in reason.message + return 'Invalid memory access' in str(reason) def _is_program_exit(reason): @@ -277,7 +277,7 @@ def _is_program_exit(reason): if not isinstance(reason, TerminateState): return False - return 'Program finished with exit status' in reason.message + return 'Program finished with exit status' in str(reason) def done_test(_, state, state_id, reason): @@ -289,7 +289,8 @@ def done_test(_, state, state_id, reason): # DeepState API, so we can just report it as is. Otherwise, we check to see if # it was due to behavior that would typically crash the program being analyzed. # If so, we save it as a crash. If not, we abandon it. - if OUR_TERMINATION_REASON not in reason: + + if str(OUR_TERMINATION_REASON) != str(reason): if _is_program_crash(reason): L.info("State {} terminated due to crashing program behavior: {}".format( state_id, reason)) @@ -330,7 +331,8 @@ def find_symbol_ea(m, name): def do_run_test(state, apis, test, hook_test=False): """Run an individual test case.""" state.cpu.PC = test.ea - m = manticore.Manticore(state, sys.argv[1:]) + m = manticore.native.Manticore(state, sys.argv[1:]) + #m = MainThreadWrapper(m, _CONTROLLER) m.verbosity(1) state = m.initial_state @@ -364,7 +366,7 @@ def do_run_test(state, apis, test, hook_test=False): m.add_hook(test.ea, hook(hook_TakeOver)) m.subscribe('will_terminate_state', done_test) - m.run() + m.run(procs=1) def run_test(state, apis, test, hook_test=False): @@ -377,7 +379,7 @@ def run_test(state, apis, test, hook_test=False): def run_tests(args, state, apis): """Run all of the test cases.""" - pool = multiprocessing.Pool(processes=max(1, args.num_workers)) + #pool = multiprocessing.Pool(processes=max(1, args.num_workers)) results = [] mc = DeepManticore(state) tests = mc.find_test_cases() @@ -386,11 +388,11 @@ def run_tests(args, state, apis): len(tests), args.num_workers)) for test in tests: - res = pool.apply_async(run_test, (state, apis, test)) + res = run_test(state, apis, test) results.append(res) - pool.close() - pool.join() + #pool.close() + #pool.join() exit(0) @@ -435,7 +437,7 @@ def main_takeover(m, args, takeover_symbol): takeover_hook = lambda state: run_test(state, apis, fake_test, hook_test) m.add_hook(takeover_ea, takeover_hook) - m.run() + m.run(procs=1) def main_unit_test(m, args): @@ -459,14 +461,14 @@ def main_unit_test(m, args): del mc m.add_hook(setup_ea, lambda state: run_tests(args, state, apis)) - m.run() + m.run(procs=1) def main(): args = DeepManticore.parse_args() try: - m = manticore.Manticore(args.binary) + m = manticore.native.Manticore(args.binary) except Exception as e: L.critical("Cannot create Manticore instance on binary {}: {}".format( args.binary, e)) diff --git a/bin/setup.py.in b/bin/setup.py.in index f4a03e1..512e09a 100644 --- a/bin/setup.py.in +++ b/bin/setup.py.in @@ -30,10 +30,7 @@ setuptools.setup( author_email="peter@trailofbits.com", license="Apache-2.0", keywords="tdd testing symbolic execution", - install_requires=[ - "claripy==7.8.6.16 ; sys_platform != 'darwin'", - "angr==7.8.7.1 ; sys_platform != 'darwin'", - "manticore==0.1.10 ; sys_platform != 'darwin'"], + install_requires=[], #'claripy==7.8.6.16','angr==7.8.7.1', 'manticore'], entry_points={ 'console_scripts': [ 'deepstate = deepstate.main_manticore:main',