From b473726781dc78f4f2b64735ee6be2f1c459596d Mon Sep 17 00:00:00 2001 From: ggrieco-tob Date: Wed, 8 Aug 2018 09:01:02 -0300 Subject: [PATCH 01/14] proof-of-concept of the experimental python3.6 version (manticore only) --- bin/deepstate/common.py | 9 +++++---- bin/deepstate/main_manticore.py | 10 +++++----- bin/setup.py.in | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index 8ca19ee..21469de 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -16,7 +16,8 @@ import logging logging.basicConfig() import argparse -import md5 +#import md5 +import hashlib import os import struct @@ -147,7 +148,7 @@ class DeepState(object): def read_c_string(self, ea, concretize=True, constrain=False): """Read a NUL-terminated string from `ea`.""" - assert isinstance(ea, (int, long)) + assert isinstance(ea, (int)) chars = [] while True: b, ea = self.read_uint8_t(ea, concretize=concretize, constrain=constrain) @@ -285,7 +286,7 @@ class DeepState(object): for b in byte_str: if isinstance(b, str): new_bytes.extend(ord(bn) for bn in b) - elif isinstance(b, (int, long)): + elif isinstance(b, (int)): new_bytes.append(b) elif isinstance(b, (list, tuple)): new_bytes.extend(self._concretize_bytes(b)) @@ -333,7 +334,7 @@ class DeepState(object): return test_dir = self.context['test_dir'] - test_name = md5.new(input_bytes).hexdigest() + test_name = hashlib.md5.new(input_bytes).hexdigest() if self.context['failed']: test_name += ".fail" diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index 8f0a356..ae000ff 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -21,7 +21,7 @@ try: import manticore except Exception as e: if "Z3NotFoundError" in repr(type(e)): - print "Manticore requires Z3 to be installed." + print("Manticore requires Z3 to be installed.") sys.exit(255) else: raise @@ -88,7 +88,7 @@ class DeepManticore(DeepState): return ea + 1 def concretize(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int)): return val elif isinstance(val, str): assert len(val) == 1 @@ -104,7 +104,7 @@ class DeepManticore(DeepState): return concrete_val def concretize_min(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int)): return val concrete_val = min(self.state.concretize(val, policy='MINMAX')) if constrain: @@ -112,7 +112,7 @@ class DeepManticore(DeepState): return concrete_val def concretize_max(self, val, constrain=False): - if isinstance(val, (int, long)): + if isinstance(val, (int)): return val concrete_val = max(self.state.concretize(val, policy='MINMAX')) if constrain: @@ -121,7 +121,7 @@ class DeepManticore(DeepState): def concretize_many(self, val, max_num): assert 0 < max_num - if isinstance(val, (int, long)): + if isinstance(val, (int)): return [val] return self.state.solve_n(val, max_num) diff --git a/bin/setup.py.in b/bin/setup.py.in index f1eca5b..1079111 100644 --- a/bin/setup.py.in +++ b/bin/setup.py.in @@ -30,11 +30,11 @@ setuptools.setup( author_email="peter@trailofbits.com", license="Apache-2.0", keywords="tdd testing symbolic execution", - install_requires=['claripy==7.8.6.16','angr==7.8.7.1', 'manticore'], + install_requires=[], #'claripy==7.8.6.16','angr==7.8.7.1', 'manticore'], entry_points={ 'console_scripts': [ 'deepstate = deepstate.main_manticore:main', - 'deepstate-angr = deepstate.main_angr:main', + #'deepstate-angr = deepstate.main_angr:main', 'deepstate-manticore = deepstate.main_manticore:main', ] }) From bb9e02834db7786eed06563885224b57d467bd74 Mon Sep 17 00:00:00 2001 From: ggrieco-tob Date: Mon, 24 Dec 2018 11:39:46 -0300 Subject: [PATCH 02/14] improved python3 support --- bin/deepstate/common.py | 12 +++++++----- bin/deepstate/main_manticore.py | 29 +++++++++++++++++------------ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index 21469de..0ead821 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -242,7 +242,7 @@ class DeepState(object): # Create the symbols that feed API functions like `DeepState_Int`. symbols = [] - for i, ea in enumerate(xrange(apis['InputBegin'], apis['InputEnd'])): + for i, ea in enumerate(range(apis['InputBegin'], apis['InputEnd'])): symbol = self.create_symbol('DEEP_INPUT_{}'.format(i), 8) self.write_uint8_t(ea, symbol) symbols.append(symbol) @@ -334,7 +334,9 @@ class DeepState(object): return test_dir = self.context['test_dir'] - test_name = hashlib.md5.new(input_bytes).hexdigest() + md5 = hashlib.md5() + md5.update(input_bytes) + test_name = md5.hexdigest() if self.context['failed']: test_name += ".fail" @@ -368,7 +370,7 @@ class DeepState(object): # likely to get the same concrete byte values across different tools (e.g. # Manticore, Angr). input_bytes = bytearray() - for i in xrange(input_length): + for i in range(input_length): b = self.concretize_min(symbols[i], constrain=True) input_bytes.append(b) @@ -464,7 +466,7 @@ class DeepState(object): begin_ea, end_ea)) self.abandon_test() - for i in xrange(end_ea - begin_ea): + for i in range(end_ea - begin_ea): val, _ = self.read_uint8_t(begin_ea + i, concretize=True, constrain=True) self.write_uint8_t(begin_ea + i, val) @@ -551,7 +553,7 @@ class DeepState(object): format_str = self.read_c_string(format_ea)[0] unpack_str = self.read_c_string(unpack_ea)[0] uint64_bytes = [] - for i in xrange(8): + for i in range(8): b, _ = self.read_uint8_t(uint64_ea + i, concretize=False) uint64_bytes.append(b) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index ae000ff..63d4e69 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -18,7 +18,8 @@ logging.basicConfig() import sys try: - import manticore + import manticore + import manticore.native except Exception as e: if "Z3NotFoundError" in repr(type(e)): print("Manticore requires Z3 to be installed.") @@ -52,7 +53,7 @@ class DeepManticore(DeepState): return manticore.issymbolic(val) def create_symbol(self, name, size_in_bits): - return self.state.new_symbolic_value(size_in_bits, name) + return self.state.new_symbolic_value(size_in_bits) def read_uintptr_t(self, ea, concretize=True, constrain=False): addr_size_bits = self.state.cpu.address_bit_size @@ -285,7 +286,10 @@ def done_test(_, state, state_id, reason): # DeepState API, so we can just report it as is. Otherwise, we check to see if # it was due to behavior that would typically crash the program being analyzed. # If so, we save it as a crash. If not, we abandon it. - if OUR_TERMINATION_REASON not in reason: + if type(reason) is not str: + reason = str(reason) + + if OUR_TERMINATION_REASON != reason: if _is_program_crash(reason): L.info("State {} terminated due to crashing program behavior: {}".format( state_id, reason)) @@ -326,7 +330,8 @@ def find_symbol_ea(m, name): def do_run_test(state, apis, test, hook_test=False): """Run an individual test case.""" state.cpu.PC = test.ea - m = manticore.Manticore(state, sys.argv[1:]) + m = manticore.native.Manticore(state, sys.argv[1:]) + #m = MainThreadWrapper(m, _CONTROLLER) m.verbosity(1) state = m.initial_state @@ -356,7 +361,7 @@ def do_run_test(state, apis, test, hook_test=False): m.add_hook(test.ea, hook(hook_TakeOver)) m.subscribe('will_terminate_state', done_test) - m.run() + m.run(procs=1) def run_test(state, apis, test, hook_test=False): @@ -369,7 +374,7 @@ def run_test(state, apis, test, hook_test=False): def run_tests(args, state, apis): """Run all of the test cases.""" - pool = multiprocessing.Pool(processes=max(1, args.num_workers)) + #pool = multiprocessing.Pool(processes=max(1, args.num_workers)) results = [] mc = DeepManticore(state) tests = mc.find_test_cases() @@ -378,11 +383,11 @@ def run_tests(args, state, apis): len(tests), args.num_workers)) for test in tests: - res = pool.apply_async(run_test, (state, apis, test)) + res = run_test(state, apis, test) results.append(res) - pool.close() - pool.join() + #pool.close() + #pool.join() exit(0) @@ -426,7 +431,7 @@ def main_takeover(m, args, takeover_symbol): takeover_hook = lambda state: run_test(state, apis, fake_test, hook_test) m.add_hook(takeover_ea, takeover_hook) - m.run() + m.run(procs=1) def main_unit_test(m, args): @@ -450,14 +455,14 @@ def main_unit_test(m, args): del mc m.add_hook(setup_ea, lambda state: run_tests(args, state, apis)) - m.run() + m.run(procs=1) def main(): args = DeepManticore.parse_args() try: - m = manticore.Manticore(args.binary) + m = manticore.native.Manticore(args.binary) except Exception as e: L.critical("Cannot create Manticore instance on binary {}: {}".format( args.binary, e)) From a3c4560d283101f973b2dd6e3b4b133d636e414b Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 09:11:57 -0300 Subject: [PATCH 03/14] Fixed python-nose --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a193f22..a466da6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ sudo: true #- "2.7" install: - sudo apt-get -y update -- sudo apt-get -y install build-essential gcc-multilib cmake python3 python3-pip python3-setuptools libffi-dev python3-nose +- sudo apt-get -y install build-essential gcc-multilib cmake python3 python3-pip python3-setuptools libffi-dev python-nose - pip3 install pyflakes --user - pip3 install angr --user - pip3 install manticore --user From 0ac3c733baeb82b6ebe52fe7e1f8397aae7178ea Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 09:28:23 -0300 Subject: [PATCH 04/14] Update .travis.yml --- .travis.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index a466da6..26625a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ sudo: true #- "2.7" install: - sudo apt-get -y update -- sudo apt-get -y install build-essential gcc-multilib cmake python3 python3-pip python3-setuptools libffi-dev python-nose +- sudo apt-get -y install build-essential gcc-multilib cmake python3 python3-pip python3-setuptools libffi-dev python3-nose - pip3 install pyflakes --user - pip3 install angr --user - pip3 install manticore --user @@ -29,15 +29,15 @@ env: script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py -- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic.py ; fi -- if [ $TASK = CRASH ]; then nosetests tests/test_crash.py ; fi -- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture.py ; fi -- if [ $TASK = KLEE ]; then nosetests tests/test_klee.py ; fi -- if [ $TASK = LISTS ]; then nosetests tests/test_lists.py ; fi -- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof.py ; fi -- if [ $TASK = RUNLEN ]; then nosetests tests/test_runlen.py ; fi -- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow.py ; fi -- if [ $TASK = PRIMES ]; then nosetests tests/test_primes.py ; fi -- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting.py ; fi -- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover.py ; fi +- if [ $TASK = ARITHMETIC ]; then nosetests3 tests/test_arithmetic.py ; fi +- if [ $TASK = CRASH ]; then nosetests3 tests/test_crash.py ; fi +- if [ $TASK = FIXTURE ]; then nosetests3 tests/test_fixture.py ; fi +- if [ $TASK = KLEE ]; then nosetests3 tests/test_klee.py ; fi +- if [ $TASK = LISTS ]; then nosetests3 tests/test_lists.py ; fi +- if [ $TASK = ONEOF ]; then nosetests3 tests/test_oneof.py ; fi +- if [ $TASK = RUNLEN ]; then nosetests3 tests/test_runlen.py ; fi +- if [ $TASK = OVERFLOW ]; then nosetests3 tests/test_overflow.py ; fi +- if [ $TASK = PRIMES ]; then nosetests3 tests/test_primes.py ; fi +- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests3 tests/test_streamingandformatting.py ; fi +- if [ $TASK = TAKEOVER ]; then nosetests3 tests/test_takeover.py ; fi From e97c5fa1ee4db08f330566ccc13e6371d84ebf75 Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 10:22:52 -0300 Subject: [PATCH 05/14] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 448dfa7..3f9328a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,7 +52,7 @@ set(CMAKE_C_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_DEBUG "-O3") set(CMAKE_CXX_FLAGS_RELEASE "-O3") -find_program(PYTHON "python2.7") +find_program(PYTHON "python3.6") # Enable the GNU extensions set(CMAKE_CXX_EXTENSIONS ON) From bddaca59d521ef932da0b351309a5bfe7a299f93 Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 10:36:10 -0300 Subject: [PATCH 06/14] Upgrade to bionic (to use python3.6 from the official repositories) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26625a5..9402fff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ language: bash -dist: xenial +dist: bionic sudo: true #python: #- "2.7" install: - sudo apt-get -y update -- sudo apt-get -y install build-essential gcc-multilib cmake python3 python3-pip python3-setuptools libffi-dev python3-nose +- sudo apt-get -y install build-essential gcc-multilib cmake python3.6 python3-pip python3-setuptools libffi-dev python3-nose - pip3 install pyflakes --user - pip3 install angr --user - pip3 install manticore --user From 5e4980c212b135ac654d6344b03e3bcdad6a4095 Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 10:50:19 -0300 Subject: [PATCH 07/14] Update .travis.yml --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9402fff..694aea9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ language: bash -dist: bionic +dist: xenial sudo: true -#python: -#- "2.7" +python: +- "3.6.5" install: - sudo apt-get -y update -- sudo apt-get -y install build-essential gcc-multilib cmake python3.6 python3-pip python3-setuptools libffi-dev python3-nose +- sudo apt-get -y install build-essential gcc-multilib cmake python3-pip python3-setuptools libffi-dev python3-nose - pip3 install pyflakes --user - pip3 install angr --user - pip3 install manticore --user From 317794bd38666c4afd5bab49edf78f821028d8c7 Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 11:15:09 -0300 Subject: [PATCH 08/14] Update .travis.yml --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 694aea9..2f18b6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,11 @@ python: install: - sudo apt-get -y update - sudo apt-get -y install build-essential gcc-multilib cmake python3-pip python3-setuptools libffi-dev python3-nose -- pip3 install pyflakes --user -- pip3 install angr --user -- pip3 install manticore --user +- pip3 install pip --user +- pip3.6 -V +- pip3.6 install pyflakes --user +- pip3.6 install angr --user +- pip3.6 install manticore --user - mkdir build - cd build - cmake .. From b676147c7af5bd41535080fc41507cdc94999c2a Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 11:16:29 -0300 Subject: [PATCH 09/14] Update .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f18b6f..95cb826 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ -language: bash dist: xenial sudo: true +language: python python: -- "3.6.5" + - 3.6.5 install: - sudo apt-get -y update - sudo apt-get -y install build-essential gcc-multilib cmake python3-pip python3-setuptools libffi-dev python3-nose From 011639cea4bd03019db165022b3740ea2fa6b4ee Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 11:21:32 -0300 Subject: [PATCH 10/14] Update .travis.yml --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 95cb826..d18d4d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,11 @@ python: install: - sudo apt-get -y update - sudo apt-get -y install build-essential gcc-multilib cmake python3-pip python3-setuptools libffi-dev python3-nose -- pip3 install pip --user +- pip3 install pip - pip3.6 -V -- pip3.6 install pyflakes --user -- pip3.6 install angr --user -- pip3.6 install manticore --user +- pip3.6 install pyflakes +- pip3.6 install angr +- pip3.6 install manticore - mkdir build - cd build - cmake .. From 3ce73c78f477cebae8bdf3b159b1ba32a67dd95a Mon Sep 17 00:00:00 2001 From: ggrieco-tob <31542053+ggrieco-tob@users.noreply.github.com> Date: Mon, 7 Jan 2019 12:24:09 -0300 Subject: [PATCH 11/14] Update main_manticore.py --- bin/deepstate/main_manticore.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index 1d10e8e..0981e2c 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -26,7 +26,6 @@ except Exception as e: sys.exit(255) else: raise -import multiprocessing import traceback from .common import DeepState, TestInfo From 3abda236018832d78ba7d10d8d1da41525df3add Mon Sep 17 00:00:00 2001 From: ggrieco-tob Date: Mon, 7 Jan 2019 12:40:40 -0300 Subject: [PATCH 12/14] fixed TerminateState identification issue --- bin/deepstate/main_manticore.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index 1d10e8e..4fad3f8 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -267,7 +267,7 @@ def _is_program_crash(reason): if not isinstance(reason, TerminateState): return False - return 'Invalid memory access' in reason.message + return 'Invalid memory access' in str(reason) def _is_program_exit(reason): @@ -278,7 +278,7 @@ def _is_program_exit(reason): if not isinstance(reason, TerminateState): return False - return 'Program finished with exit status' in reason.message + return 'Program finished with exit status' in str(reason) def done_test(_, state, state_id, reason): @@ -290,10 +290,8 @@ def done_test(_, state, state_id, reason): # DeepState API, so we can just report it as is. Otherwise, we check to see if # it was due to behavior that would typically crash the program being analyzed. # If so, we save it as a crash. If not, we abandon it. - if type(reason) is not str: - reason = str(reason) - if OUR_TERMINATION_REASON != reason: + if str(OUR_TERMINATION_REASON) != str(reason): if _is_program_crash(reason): L.info("State {} terminated due to crashing program behavior: {}".format( state_id, reason)) From 19df0d94d1f82fcc1c131e54e1bf65d7d320069b Mon Sep 17 00:00:00 2001 From: Gustavo Grieco <31542053+ggrieco-tob@users.noreply.github.com> Date: Thu, 24 Jan 2019 14:09:35 -0300 Subject: [PATCH 13/14] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d18d4d8..c33e1e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - pip3.6 -V - pip3.6 install pyflakes - pip3.6 install angr -- pip3.6 install manticore +- pip3.6 install manticore[native] - mkdir build - cd build - cmake .. From 151354cb90547b59fb19b880fe4467d015993544 Mon Sep 17 00:00:00 2001 From: Gustavo Grieco <31542053+ggrieco-tob@users.noreply.github.com> Date: Thu, 24 Jan 2019 14:17:50 -0300 Subject: [PATCH 14/14] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c33e1e6..bfed9b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - pip3.6 -V - pip3.6 install pyflakes - pip3.6 install angr -- pip3.6 install manticore[native] +- pip3.6 install https://github.com/trailofbits/manticore/archive/master.zip - mkdir build - cd build - cmake ..