Better plugin context management. (#730)

* Better plugin context management. Example UseDef logging plugin for evm

* Example plugin to log use-def

* Better plugin name checking hints

* Better plugin name checking hints2

* Fix uninititalize-plugins (found via the warning)

* Remove redundant variables in example

* Typo

* Better warning print (class name)

* better safety check + context fix

* better variabke naming

* Update manticore.py

examples/evm/use_def.py

@@ -0,0 +1,110 @@
+from manticore.ethereum import ManticoreEVM, Plugin
+from manticore.core.smtlib import solver
+from manticore.core.smtlib.visitors import arithmetic_simplifier, pretty_print, constant_folder
+################ Script #######################
+
+m = ManticoreEVM()
+m.verbosity(0)
+#And now make the contract account to analyze
+# cat  | solc --bin 
+source_code = '''
+pragma solidity ^0.4;
+contract C {
+    uint c;
+    bool enabled;
+    bool i;
+    function C() public {
+        c =0;
+        enabled = false;
+        i = false;
+        
+    }
+    function f1() public {
+        c+=1;
+    }
+    function f2() public {
+        if(c>100)
+            enabled=true;
+        
+    }
+    function f3() public{
+        if (!enabled) 
+            return;
+        i = true;
+        
+    }
+}
+'''
+print source_code
+class EVMUseDef(Plugin):
+    def _get_concrete_hex(self, state, array):
+        r = ''
+        for i in array:
+            l = state.solve_n(i, 2)
+            if len(l) == 1:
+                r += '%02x'%l[0]
+        if len(r) != 8:
+            return
+        return r
+
+
+    def did_evm_write_storage_callback(self, state, offset, value, **kwargs):
+        m = self.manticore
+        world = state.platform
+        tx = world.all_transactions[-1]
+        md = m.get_metadata(tx.address)
+
+        r = self._get_concrete_hex(state, tx.data[0:4])
+        if r is None:
+            return
+
+
+        offsets = state.solve_n(offset, 3000)
+        with self.locked_context('storage_writes', dict) as storage_writes:
+            contract_function = (md.name, md.get_func_name(r))
+            if contract_function not in storage_writes:
+                storage_writes[contract_function] = set()
+            for off in offsets:
+                storage_writes[contract_function].add(off)
+
+    def did_evm_read_storage_callback(self, state, offset, value, **kwargs):
+        m = self.manticore
+        world = state.platform
+        tx = world.all_transactions[-1]
+        md = m.get_metadata(tx.address)
+
+        r = self._get_concrete_hex(state, tx.data[0:4])
+        if r is None:
+            return
+
+        offsets = state.solve_n(offset, 3000)
+        with self.locked_context('storage_reads', dict) as storage_reads:
+            contract_function = (md.name, md.get_func_name(r))
+            if contract_function not in storage_reads:
+                storage_reads[contract_function] = set()
+            for off in offsets:
+                storage_reads[contract_function].add(off)
+
+#Initialize accounts
+user_account = m.create_account(balance=1000)
+contract_account = m.solidity_create_contract(source_code, owner=user_account)
+p = EVMUseDef()
+m.register_plugin(p)
+
+
+symbolic_data = m.make_symbolic_buffer(320)
+symbolic_value = m.make_symbolic_value()
+m.transaction(  caller=user_account,
+                   address=contract_account,
+                   value=symbolic_value,
+                   data=symbolic_data
+                 )
+print "READS", p.context['storage_reads']
+print "WRITES", p.context['storage_writes']
+
+print "It makes no sense to try f3() after 1 tx"
+
+m.finalize()
+print "[+] Look for results in %s"% m.workspace
+
+

manticore/__main__.py

@@ -80,15 +80,15 @@ def parse_arguments():
 
 
 def ethereum_cli(args):
-    from ethereum import ManticoreEVM, IntegerOverflow, UnitializedStorage, UnitializedMemory
+    from ethereum import ManticoreEVM, IntegerOverflow, UninitializedStorage, UninitializedMemory
     log.init_logging()
 
     m = ManticoreEVM(procs=args.procs)
 
     ################ Default? Detectors #######################
     m.register_detector(IntegerOverflow())
-    m.register_detector(UnitializedStorage())
+    m.register_detector(UninitializedStorage())
-    m.register_detector(UnitializedMemory())
+    m.register_detector(UninitializedMemory())
 
     logger.info("Beginning analysis")
 

manticore/core/plugin.py

@@ -3,10 +3,35 @@ import logging
 from capstone import CS_GRP_JUMP
 
 from ..utils.helpers import issymbolic
+from contextlib import contextmanager
+
 logger = logging.getLogger(__name__)
 
 
 class Plugin(object):
+    @contextmanager
+    def locked_context(self, key=None, value_type=list):
+        """
+        A context manager that provides safe parallel access to the global Manticore context.
+        This should be used to access the global Manticore context
+        when parallel analysis is activated. Code within the `with` block is executed
+        atomically, so access of shared variables should occur within.
+        """
+        plugin_context_name = str(type(self))
+        with self.manticore.locked_context(plugin_context_name, dict) as context:
+            assert value_type in (list, dict, set)
+            ctx = context.get(key, value_type())
+            yield ctx
+            context[key] = ctx
+
+    @property
+    def context(self):
+        ''' Convenient access to shared context '''
+        plugin_context_name = str(type(self))
+        if plugin_context_name not in self.manticore.context:
+            self.manticore.context[plugin_context_name] = {}
+        return self.manticore.context[plugin_context_name]
+
     def __init__(self): 
         self.manticore = None
         self.last_reg_state = {}

manticore/ethereum.py

@@ -64,11 +64,11 @@ class IntegerOverflow(Detector):
             if state.can_be_true(arguments[1] > arguments[0]):
                 self.add_finding(state, "Integer underflow at {} instruction".format(mnemonic))
             
-class UnitializedMemory(Detector):
+class UninitializedMemory(Detector):
     '''
         Detects uses of uninitialized memory
     '''
-    def did_evm_read_memory(self, state, offset, value):
+    def did_evm_read_memory_callback(self, state, offset, value):
         if not state.can_be_true(value != 0):
             #Not initialized memory should be zero
             return 
@@ -80,16 +80,16 @@ class UnitializedMemory(Detector):
         if state.can_be_true(cbu):
             self.add_finding(state, "Potentially reading uninitialized memory at instruction")
  
-    def did_evm_write_memory(self, state, offset, value):
+    def did_evm_write_memory_callback(self, state, offset, value):
         #concrete or symbolic write
         state.context.setdefault('seth.detectors.initialized_memory',set()).add(offset)
 
 
-class UnitializedStorage(Detector):
+class UninitializedStorage(Detector):
     '''
         Detects uses of uninitialized storage
     '''
-    def did_evm_read_storage(self, state, offset, value):
+    def did_evm_read_storage_callback(self, state, offset, value):
         if not state.can_be_true(value != 0):
             #Not initialized memory should be zero
             return 
@@ -101,7 +101,7 @@ class UnitializedStorage(Detector):
         if state.can_be_true(cbu):
             self.add_finding(state, "Potentially reading uninitialized storage")
  
-    def did_evm_write_storage(self, state, offset, value):
+    def did_evm_write_storage_callback(self, state, offset, value):
         #concrete or symbolic write
         state.context.setdefault('seth.detectors.initialized_storage',set()).add(offset)
 

manticore/manticore.py

@@ -212,13 +212,26 @@ class Manticore(Eventful):
             if callback is not None:
                 self.subscribe(event_name, callback)
 
-        if logger.isEnabledFor(logging.DEBUG):
+        #Safety checks
         for callback_name in dir(plugin):
             if callback_name.endswith('_callback'):
                 event_name = callback_name[:-9]
                 if event_name not in all_events:
-                        logger.warning("There is no event name %s for callback on plugin type %s", event_name, type(plugin) )
+                    logger.warning("There is no event named %s for callback on plugin %s", event_name, type(plugin).__name__ )
 
+        for event_name in all_events:
+            for plugin_method_name in dir(plugin):
+                if event_name in plugin_method_name:
+                    if not plugin_method_name.endswith('_callback') :
+                        if plugin_method_name.startswith('on_') or \
+                           plugin_method_name.startswith('will_') or \
+                           plugin_method_name.startswith('did_'):
+                            logger.warning("Plugin methods named '%s()' should end with '_callback' on plugin %s", plugin_method_name, type(plugin).__name__ )
+                    if plugin_method_name.endswith('_callback') and \
+                        not plugin_method_name.startswith('on_') and \
+                        not plugin_method_name.startswith('will_') and \
+                        not plugin_method_name.startswith('did_'):
+                            logger.warning("Plugin methods named '%s()' should start with 'on_', 'will_' or 'did_' on plugin %s", plugin_method_name, type(plugin).__name__)
 
 
     def unregister_plugin(self, plugin):

manticore/platforms/evm.py

@@ -44,6 +44,9 @@ class Transaction(object):
         ''' Implements serialization/pickle '''
         return (self.__class__,  (self.sort,  self.address, self.origin, self.price, self.data, self.caller, self.value, self.return_data, self.result))
 
+    def __str__(self):
+        return 'Transaction(%s, from=0x%x, to=0x%x, value=%r, data=%r..)' %(self.sort, self.caller, self.address, self.value, self.data)
+
 class EVMLog():
     def __init__(self, address, memlog, topics):
         self.address = address
@@ -1679,12 +1682,14 @@ class EVM(Eventful):
 
     def SLOAD(self, offset):
         '''Load word from storage'''
+        self._publish('will_evm_read_storage', offset)
         value = self.global_storage[self.address]['storage'].get(offset,0)
         self._publish('did_evm_read_storage', offset, value)
         return value
 
     def SSTORE(self, offset, value):
         '''Save word to storage'''
+        self._publish('will_evm_write_storage', offset, value)
         self.global_storage[self.address]['storage'][offset] = value
         if value is 0:
             del self.global_storage[self.address]['storage'][offset]
@@ -1893,6 +1898,7 @@ class EVMWorld(Platform):
         self._sha3 = {}
         self._pending_transaction = None
         self._transactions = list()
+        self._internal_transactions = list()
 
     def __getstate__(self):
         state = super(EVMWorld, self).__getstate__()
@@ -1904,7 +1910,7 @@ class EVMWorld(Platform):
         state['callstack'] = self._callstack
         state['deleted_address'] = self._deleted_address
         state['transactions'] = self._transactions
-
+        state['internal_transactions'] = self._internal_transactions
         return state
 
     def __setstate__(self, state):
@@ -1917,6 +1923,7 @@ class EVMWorld(Platform):
         self._callstack = state['callstack']
         self._deleted_address = state['deleted_address']
         self._transactions = state['transactions']
+        self._internal_transactions = state['internal_transactions']
         self._do_events()
 
     def _do_events(self):
@@ -1949,6 +1956,23 @@ class EVMWorld(Platform):
     def transactions(self):
         return self._transactions
 
+    @property
+    def internal_transactions(self):
+        number_of_transactions = len(self._transactions)
+        for _ in range( len (self._internal_transactions), number_of_transactions):
+            self._internal_transactions.append([])
+        return self._internal_transactions
+
+    @property
+    def all_transactions(self):
+        txs = []
+        for tx in self._transactions:
+            txs.append(tx)
+            for txi in self.internal_transactions[self._transactions.index(tx)]:
+                txs.append(txi)
+        return txs
+
+
     @property
     def last_return_data(self):
         return self.transactions[-1].return_data
@@ -2318,6 +2342,8 @@ class EVMWorld(Platform):
         new_vm = EVM(self._constraints, address, origin, price, data, caller, value, bytecode, header, global_storage=self.storage)
         self._push_vm(new_vm)
 
+
+        tx = Transaction(ty, address, origin, price, data, caller, value, None, None)
         if is_human_tx:
             #handle human transactions
             if ty == 'Create':
@@ -2325,8 +2351,12 @@ class EVMWorld(Platform):
             elif ty == 'Call':
                 self.current.last_exception = Call(None, None, None, None)
 
-            tx = Transaction(ty, address, origin, price, data, caller, value, None, None)
             self._transactions.append(tx)
+        else:            
+            n = len(self._transactions)
+            if len (self._internal_transactions) < n:
+                self._internal_transactions.append([])
+            self._internal_transactions[n].append(tx)
 
 
     def CALL(self, gas, to, value, data):