Port to real python package structure (#6)

* Initial moving

* Initial package __init__.py

* Use relative imports

* Correct imports for tests

* Rename main.py -> __main__.py

* Use relative imports in main

* Refactor cli to be used with setuptools

* Add basic setup.py

* Fix test_memory import

* Fix test_cpu_manual imports

* Correct iterpickle import

* Use manticore console_script installed by setuptools

* Make travis install our package for integration cli tests

* Refactor model hooks dynamic import for being in pkg

* Correct import

* Rm empty line

* Fix coverage test paths

.travis.yml

@@ -20,6 +20,7 @@ install:
 - travis_retry pip install -r requirements.txt
 - travis_retry pip install --no-binary keystone-engine keystone-engine
 - travis_retry pip install coverage
+- travis_retry pip install .
 - scripts/travis_install_z3.sh
 - scripts/travis_install_unicorn.sh 
 script:

manticore/__init__.py

@@ -0,0 +1 @@
+from .manticore import Manticore, issymbolic

manticore/__main__.py

@@ -70,7 +70,7 @@ def parse_arguments():
 
 logger = logging.getLogger('MAIN')
 
-if __name__ == '__main__':
+def main():
     args = parse_arguments()
 
     m = Manticore(args.programs[0], args.programs[1:])
@@ -149,3 +149,5 @@ if __name__ == '__main__':
 
     m.dump_stats()
 
+if __name__ == '__main__':
+    main()

manticore/core/cpu/abstractcpu.py

@@ -5,9 +5,8 @@ from unicorn import *
 from unicorn.x86_const import *
 from unicorn.arm_const import *
 from abc import ABCMeta, abstractmethod
-from core.smtlib import Expression, Bool, BitVec, Array, Operators, Constant
-from core.memory import MemoryException
-from utils import gdb
+from ..smtlib import Expression, Bool, BitVec, Array, Operators, Constant
+from ..memory import MemoryException
 import sys
 from functools import wraps
 import types

manticore/core/cpu/arm.py

@@ -1,10 +1,11 @@
 import struct
 import sys
-from abstractcpu import Cpu, RegisterFile, Operand
-from abstractcpu import SymbolicPCException, InvalidPCException, Interruption
-from abstractcpu import instruction as abstract_instruction
-from register import Register
-from core.smtlib import Operators, Expression
+from .abstractcpu import Cpu, RegisterFile, Operand
+from .abstractcpu import SymbolicPCException, InvalidPCException, Interruption
+from .abstractcpu import instruction as abstract_instruction
+from .register import Register
+from ..smtlib import Operators, Expression
+# from ..smtlib import *
 from functools import wraps
 from bitwise import *
 
@@ -23,7 +24,6 @@ except:
 MU = None
 
 
-from core.smtlib import *
 
 # Custom Constants (avoid conflicts with capstone's arm constants)
 ARM_REG_APSR_N = 1000

manticore/core/cpu/bitwise.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
-from core.smtlib import Operators
-from core.smtlib.expression import BitVec
+from ..smtlib import Operators
+from ..smtlib.expression import BitVec
 
 def Mask(width):
     return (1 << width) - 1

manticore/core/cpu/cpufactory.py

@@ -1,5 +1,5 @@
-from core.cpu.x86 import AMD64Cpu, I386Cpu
-from core.cpu.arm import Armv7Cpu
+from .x86 import AMD64Cpu, I386Cpu
+from .arm import Armv7Cpu
 
 class CpuFactory(object):
     _cpus = {

manticore/core/cpu/register.py

@@ -1,4 +1,4 @@
-from core.smtlib import Operators, Expression, BitVec, Bool
+from ..smtlib import Operators, Expression, BitVec, Bool
 
 class Register(object):
     '''Generic variable width register. For 1 bit registers, allows writes

manticore/core/cpu/x86.py

@@ -27,15 +27,15 @@
 
 from capstone import *
 from capstone.x86 import *
-from abstractcpu import Cpu, RegisterFile, Operand, SANE_SIZES, instruction
-from abstractcpu import SymbolicPCException, InvalidPCException, Interruption, Sysenter, Syscall, ConcretizeRegister, ConcretizeArgument
+from .abstractcpu import Cpu, RegisterFile, Operand, SANE_SIZES, instruction
+from .abstractcpu import SymbolicPCException, InvalidPCException, Interruption, Sysenter, Syscall, ConcretizeRegister, ConcretizeArgument
 import sys
 import struct
 import types
 import weakref
 from functools import wraps, partial
 import collections
-from core.smtlib import *
+from ..smtlib import *
 from ..memory import MemoryException
 import logging
 logger = logging.getLogger("CPU")

manticore/core/executor.py

@@ -40,12 +40,12 @@ except:
     import StringIO
 from math import ceil, log
 
-from utils.nointerrupt import DelayedKeyboardInterrupt
-from cpu.abstractcpu import ConcretizeRegister, ConcretizeMemory, \
+from ..utils.nointerrupt import DelayedKeyboardInterrupt
+from .cpu.abstractcpu import ConcretizeRegister, ConcretizeMemory, \
         InvalidPCException, IgnoreAPI
-from memory import MemoryException, SymbolicMemoryException
-from smtlib import solver, Expression, Operators, SolverException, Array, BitVec, Bool, ConstraintSet
-from utils.event import Signal
+from .memory import MemoryException, SymbolicMemoryException
+from .smtlib import solver, Expression, Operators, SolverException, Array, BitVec, Bool, ConstraintSet
+from ..utils.event import Signal
 
 #Multiprocessing
 from multiprocessing import Manager
@@ -463,7 +463,7 @@ class Executor(object):
             except RuntimeError:
                 # there recursion limit exceeded problem, 
                 # try a slower, iterative solution
-                from utils import iterpickle
+                from ..utils import iterpickle
                 logger.info("WARNING: using iterpickle to dump state")
                 f.write(iterpickle.dumps(state, 2))
 
@@ -659,7 +659,7 @@ class Executor(object):
             except RuntimeError:
                 # there recursion limit exceeded problem, 
                 # try a slower, iterative solution
-                from utils import iterpickle
+                from ..utils import iterpickle
                 logger.info("WARNING: using iterpickle to dump state")
                 f.write(iterpickle.dumps(state, 2))
             f.flush()

manticore/core/memory.py

@@ -28,9 +28,9 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
 from weakref import WeakValueDictionary
 from cStringIO import StringIO
-from smtlib import *
+from .smtlib import *
 import logging
-from mappings import _mmap, _munmap
+from .mappings import _mmap, _munmap
 
 logger = logging.getLogger('MEMORY')
 

manticore/manticore.py

@@ -13,15 +13,13 @@ from multiprocessing import Process
 from elftools.elf.elffile import ELFFile
 from elftools.elf.sections import SymbolTableSection
 
-from core.executor import Executor, State, AbandonState
-from core.parser import parse
-from core.smtlib import solver, Expression, Operators, SolverException, Array
+from .core.executor import Executor, State, AbandonState
+from .core.parser import parse
+from .core.smtlib import solver, Expression, Operators, SolverException, Array, ConstraintSet
 from core.smtlib import BitVec, Bool
-from models import linux, decree, windows
+from .models import linux, decree, windows
 from utils import gdb, qemu
 
-from core.smtlib import ConstraintSet
-
 
 logger = logging.getLogger('MANTICORE')
 
@@ -475,7 +473,7 @@ class Manticore(object):
     def apply_model_hooks(self, path):
         #TODO(yan): Simplify the partial function application
 
-        # Imported straight from main.py; this will be re-written once the new
+        # Imported straight from __main__.py; this will be re-written once the new
         # event code is in place.
         import core.cpu
         import importlib
@@ -487,7 +485,7 @@ class Manticore(object):
                 cc = getattr(core.cpu.x86.ABI, cc_name)
                 fmodel = models
                 name_parts = name.split('.')
-                importlib.import_module("models.{}".format(name_parts[0]))
+                importlib.import_module(".models.{}".format(name_parts[0]), 'manticore')
                 for n in name_parts:
                     fmodel = getattr(fmodel,n)
                 assert fmodel != models

manticore/models/decree.py

@@ -2,18 +2,18 @@ import cgcrandom
 import weakref
 import sys, os, struct
 # TODO use cpu factory
-from core.cpu.x86 import I386Cpu
-from core.cpu.abstractcpu import Interruption, Syscall, ConcretizeRegister
-from core.memory import SMemory32
-from core.smtlib import *
-from binary import CGCElf
-from binary import CGCGrr
+from ..core.cpu.x86 import I386Cpu
+from ..core.cpu.abstractcpu import Interruption, Syscall, ConcretizeRegister
+from ..core.memory import SMemory32
+from ..core.smtlib import *
+from ..core.executor import SyscallNotImplemented, ProcessExit, Deadlock, RestartSyscall
+logger = logging.getLogger("DECREE")
+from ..binary import CGCElf
+from ..binary import CGCGrr
 from contextlib import closing
 import StringIO
 import logging
 import random
-from core.executor import SyscallNotImplemented, ProcessExit, Deadlock, RestartSyscall
-logger = logging.getLogger("DECREE")
 
 
 class SymbolicSyscallArgument(ConcretizeRegister):

manticore/models/libc.py

@@ -2,14 +2,13 @@ import sys, os, struct
 import StringIO
 import logging
 import random
-from core.smtlib import solver, Expression, Operators
-from core.cpu.x86 import x86Cpu
+from ..core.smtlib import solver, Expression, Operators
 #, Interruption, Syscall, ConcretizeRegister, ConcretizeMemory, ConcretizeArgument, IgnoreAPI
-from core.cpu.abstractcpu import Interruption, Syscall, \
+from ..core.cpu.abstractcpu import Interruption, Syscall, \
         ConcretizeRegister, ConcretizeArgument, IgnoreAPI, \
         ConcretizeMemory
-from core.memory import MemoryException
-from core.executor import ForkState
+from ..core.memory import MemoryException
+from ..core.executor import ForkState
 
 logger = logging.getLogger("LIBC")
 

manticore/models/linux.py

@@ -1,18 +1,18 @@
 import cgcrandom
 import weakref
 import sys, os, struct
-from utils import qemu
-from core.cpu.abstractcpu import Interruption, Syscall, ConcretizeRegister, InvalidPCException
-from core.cpu.cpufactory import CpuFactory
-from core.memory import SMemory32, SMemory64, Memory32, Memory64
-from core.smtlib import Expression, Operators, ConstraintSet
+from ..utils import qemu
+from ..core.cpu.abstractcpu import Interruption, Syscall, ConcretizeRegister, InvalidPCException
+from ..core.cpu.cpufactory import CpuFactory
+from ..core.memory import SMemory32, SMemory64, Memory32, Memory64
+from ..core.smtlib import Expression, Operators, ConstraintSet
 from elftools.elf.elffile import ELFFile
 from contextlib import closing
 import StringIO
 import logging
 import random
-from core.cpu.arm import *
-from core.executor import SyscallNotImplemented, ProcessExit
+from ..core.cpu.arm import *
+from ..core.executor import SyscallNotImplemented, ProcessExit
 logger = logging.getLogger("LINUX")
 
 class RestartSyscall(Exception):

manticore/models/windows.py

@@ -1,15 +1,15 @@
 import cgcrandom
 import weakref
 import sys, os, struct
-from core.memory import Memory, MemoryException, SMemory32
-from core.smtlib import Expression, Operators, solver
+from ..core.memory import Memory, MemoryException, SMemory32
+from ..core.smtlib import Expression, Operators, solver
 # TODO use cpu factory
-from core.cpu.x86 import I386Cpu, Sysenter
-from core.cpu.abstractcpu import Interruption, Syscall, \
+from ..core.cpu.x86 import I386Cpu, Sysenter
+from ..core.cpu.abstractcpu import Interruption, Syscall, \
         ConcretizeRegister, ConcretizeArgument, IgnoreAPI
-from core.executor import ForkState, SyscallNotImplemented
+from ..core.executor import ForkState, SyscallNotImplemented
 
-from binary.pe import minidump
+from ..binary.pe import minidump
 
 from contextlib import closing
 import StringIO

scripts/travis_test.sh

@@ -34,8 +34,8 @@ measure_cov() {
 
 #coverage report
 echo "Measuring code coverage..."
-measure_cov "core/smtlib/*" 80
-measure_cov "core/cpu/x86.py" 50
-measure_cov "core/memory.py" 85
+measure_cov "manticore/core/smtlib/*" 80
+measure_cov "manticore/core/cpu/x86.py" 50
+measure_cov "manticore/core/memory.py" 85
 
 exit ${RV}

setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='Manticore',
+    version='0.1.0',
+    packages=find_packages(),
+    entry_points={
+        'console_scripts': [
+            'manticore = manticore.__main__:main'
+        ]
+    }
+)

test/mockmem.py

@@ -1,4 +1,4 @@
-from core.smtlib import Operators
+from manticore.core.smtlib import Operators
 
 class Memory:  #todo Mock
     def getchar(self, addr):

test/test_allinsts.py

@@ -1,9 +1,9 @@
 
 import unittest
 import functools
-from core.cpu.x86 import *
-from core.smtlib import *
-from core.memory import *
+from manticore.core.cpu.x86 import *
+from manticore.core.smtlib import *
+from manticore.core.memory import *
 
 def skipIfNotImplemented(f):
     # XXX(yan) the inner function name must start with test_

test/test_armv7_bitwise.py

@@ -1,6 +1,6 @@
 import unittest
  
-from core.cpu import bitwise
+from manticore.core.cpu import bitwise
 
 
 class Armv7RF(unittest.TestCase):

test/test_armv7cpu.py

@@ -2,9 +2,9 @@ import unittest
 import struct
 from functools import wraps
 
-from core.cpu.arm import Armv7Cpu as cpu
-from core.cpu.arm import *
-from core.memory import Memory32
+from manticore.core.cpu.arm import Armv7Cpu as cpu
+from manticore.core.cpu.arm import *
+from manticore.core.memory import Memory32
 
 from capstone.arm import *
 from keystone import Ks, KS_ARCH_ARM, KS_MODE_ARM

test/test_armv7rf.py

@@ -1,7 +1,7 @@
 import unittest
  
-from core.cpu.arm import Armv7RegisterFile as RF
-from core.cpu.arm import *
+from manticore.core.cpu.arm import Armv7RegisterFile as RF
+from manticore.core.cpu.arm import *
 
 from capstone.arm import *
 

test/test_cpu_automatic.py

@@ -1,8 +1,8 @@
 
 import unittest
-from core.cpu.x86 import *
-from core.smtlib import *
-from core.memory import *
+from manticore.core.cpu.x86 import *
+from manticore.core.smtlib import *
+from manticore.core.memory import *
 
 class CPUTest(unittest.TestCase):
     class ROOperand(object):

test/test_cpu_manual.py

@@ -1,7 +1,7 @@
 import unittest
-from core.cpu.x86 import *
-from core.smtlib import Operators
-from core.memory import *
+from manticore.core.cpu.x86 import *
+from manticore.core.smtlib import Operators
+from manticore.core.memory import *
 import mockmem
 
 class ROOperand(object):

test/test_driver.py

@@ -10,7 +10,7 @@ import collections
 import time
 
 from manticore import Manticore, issymbolic
-from core.smtlib import BitVecVariable
+from manticore.core.smtlib import BitVecVariable
 
 class ManticoreDriver(unittest.TestCase):
     def setUp(self):

test/test_linux.py

@@ -1,6 +1,6 @@
 import unittest
 
-from models import linux
+from manticore.models import linux
 
 
 class LinuxTest(unittest.TestCase):

test/test_memdumps.py

@@ -87,13 +87,12 @@ class IntegrationTest(unittest.TestCase):
 
         params = self._getDumpParams(jsonfile)
 
-        SE = os.path.join(dirname, '../main.py')
         workspace = os.path.join(self.test_dir, 'ws_{}'.format(dumpname))
         logfile = os.path.join(workspace, "output.log")
 
         dumpfile = os.path.join(dumpdir, params['dump'])
 
-        args = ['python', SE, '--workspace', workspace, '--log', logfile, dumpfile]
+        args = ['manticore', '--workspace', workspace, '--log', logfile, dumpfile]
 
         for k,v in params.iteritems():
             if k.startswith("--"):

test/test_memory.py

@@ -26,13 +26,13 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from cStringIO import StringIO
-from core.smtlib import Solver, Operators
+from manticore.core.smtlib import Solver, Operators
 import unittest
 import tempfile, os
 import gc, pickle
 import fcntl
 import resource
-from core.memory import *
+from manticore.core.memory import *
 
 def issymbolic(value):
     return isinstance(value, Expression)

test/test_register.py

@@ -1,6 +1,6 @@
 import unittest
 
-from core.cpu.register import Register
+from manticore.core.cpu.register import Register
 
 class RegisterTest(unittest.TestCase):
     def test_rd(self):

test/test_signal.py

@@ -1,7 +1,7 @@
 
 import unittest
 
-from utils.event import Signal
+from manticore.utils.event import Signal
 
 class Sender(object):
     def __init__(self):

test/test_smtlibv2.py

@@ -25,7 +25,7 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from core.smtlib import *
+from manticore.core.smtlib import *
 import unittest
 import fcntl
 import resource

test/test_state.py

@@ -1,9 +1,9 @@
 import unittest
 
-from core.executor import State
-from core.smtlib import BitVecVariable
-from core.smtlib import ConstraintSet
-from models import linux
+from manticore.core.executor import State
+from manticore.core.smtlib import BitVecVariable
+from manticore.core.smtlib import ConstraintSet
+from manticore.models import linux
 
 class FakeMemory(object):
     def __init__(self):