Pythonic logger naming (#511)

* More generic logging

* Clean up logger name gen

* Refactor name summarization

manticore/core/cpu/abstractcpu.py

@@ -18,8 +18,8 @@ from ...utils.helpers import issymbolic
 from ...utils.emulate import UnicornEmulator
 from ...utils.event import Eventful
 
-logger = logging.getLogger("CPU")
-register_logger = logging.getLogger("REGISTERS")
+logger = logging.getLogger(__name__)
+register_logger = logging.getLogger('{}.registers'.format(__name__))
 
 ###################################################################################
 #Exceptions

manticore/core/cpu/arm.py

@@ -18,7 +18,7 @@ from .register import Register
 from ..smtlib import Operators, Expression, BitVecConstant
 from ...utils.helpers import issymbolic
 
-logger = logging.getLogger("CPU")
+logger = logging.getLogger(__name__)
 
 # map different instructions to a single impl here
 OP_NAME_MAP = {

manticore/core/cpu/binja.py

@@ -16,9 +16,8 @@ from ...core.cpu.disasm import BinjaILDisasm
 from ..smtlib import Operators, BitVecConstant, operator
 from ...utils.helpers import issymbolic
 
-logger = logging.getLogger("CPU")
-register_logger = logging.getLogger("REGISTERS")
-
+logger = logging.getLogger(__name__)
+register_logger = logging.getLogger('{}.registers'.format(__name__))
 
 class BinjaRegisterFile(RegisterFile):
 

manticore/core/cpu/x86.py

@@ -16,8 +16,7 @@ from ..smtlib import Operators, BitVec, Bool, BitVecConstant, operator, visitors
 from ..memory import MemoryException
 from ...utils.helpers import issymbolic
 
-logger = logging.getLogger("CPU")
-
+logger = logging.getLogger(__name__)
 
 OP_NAME_MAP = {
     'JNE':      'JNZ',

manticore/core/executor.py

@@ -23,7 +23,7 @@ def mgr_init():
 manager = SyncManager()
 manager.start(mgr_init)
 
-logger = logging.getLogger("EXECUTOR")
+logger = logging.getLogger(__name__)
 
 
 def sync(f):

manticore/core/memory.py

@@ -6,7 +6,7 @@ import logging
 from ..utils.mappings import _mmap, _munmap
 from ..utils.helpers import issymbolic
 
-logger = logging.getLogger('MEMORY')
+logger = logging.getLogger(__name__)
 
 
 class MemoryException(Exception):

manticore/core/smtlib/__init__.py

@@ -7,7 +7,7 @@ import math
 
 
 import logging
-logger = logging.getLogger("SMT")
+logger = logging.getLogger(__name__)
 
 '''
 class OperationNotPermited(SolverException):

manticore/core/smtlib/constraints.py

@@ -1,7 +1,8 @@
 from expression import BitVecVariable, BoolVariable, ArrayVariable, Array, Bool, BitVec, BitVecConstant, BoolConstant, ArrayProxy
 from visitors import GetDeclarations, TranslatorSmtlib, ArithmeticSimplifier, PrettyPrinter, pretty_print, translate_to_smtlib, get_depth, get_variables, arithmetic_simplifier
 import logging, weakref
-logger = logging.getLogger('SMT')
+
+logger = logging.getLogger(__name__)
 
 class ConstraintSet(object):
     ''' Constraint Sets

manticore/core/smtlib/solver.py

@@ -26,7 +26,7 @@ from visitors import *
 from ...utils.helpers import issymbolic, memoized
 import collections
 
-logger = logging.getLogger("SMT")
+logger = logging.getLogger(__name__)
 
 class Z3NotFoundError(EnvironmentError):
     pass

manticore/core/smtlib/visitors.py

@@ -1,7 +1,7 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
 from expression import *
 import logging
-logger = logging.getLogger("VISITOR")
+logger = logging.getLogger(__name__)
 
 
 class Visitor(object):

manticore/core/state.py

@@ -12,7 +12,7 @@ from .cpu.abstractcpu import ConcretizeRegister
 from .memory import ConcretizeMemory, MemoryException
 from ..platforms.platform import *
 
-logger = logging.getLogger("STATE")
+logger = logging.getLogger(__name__)
 
 class StateException(Exception):
     ''' All state related exceptions '''

manticore/core/workspace.py

@@ -16,7 +16,7 @@ from multiprocessing.managers import SyncManager
 from .smtlib import solver
 from .smtlib.solver import SolverException
 
-logger = logging.getLogger('WORKSPACE')
+logger = logging.getLogger(__name__)
 
 manager = SyncManager()
 manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))

manticore/manticore.py

@@ -27,8 +27,9 @@ from .utils.nointerrupt import WithKeyboardInterruptAs
 import logging
 from .utils import log
 
+
+logger = logging.getLogger(__name__)
 log.init_logging()
-logger = logging.getLogger('MANTICORE')
 
 def make_binja(program, disasm, argv, env, symbolic_files, concrete_start=''):
     def _check_disassembler_present(disasm):

manticore/platforms/decree.py

@@ -15,7 +15,7 @@ import StringIO
 import logging
 import random
 
-logger = logging.getLogger("PLATFORM")
+logger = logging.getLogger(__name__)
 
 class RestartSyscall(Exception):
     pass

manticore/platforms/linux.py

@@ -21,7 +21,7 @@ from ..platforms.platform import Platform
 from ..utils.helpers import issymbolic, is_binja_disassembler
 from . import linux_syscalls
 
-logger = logging.getLogger("PLATFORM")
+logger = logging.getLogger(__name__)
 
 class RestartSyscall(Exception):
     pass

manticore/platforms/windows.py

@@ -17,7 +17,7 @@ import StringIO
 import logging
 import random
 from windows_syscalls import syscalls_num
-logger = logging.getLogger("PLATFORM")
+logger = logging.getLogger(__name__)
 
 
 class SyscallNotImplemented(TerminateState):

manticore/utils/emulate.py

@@ -15,7 +15,7 @@ from capstone import *
 from capstone.arm import *
 from capstone.x86 import *
 
-logger = logging.getLogger("EMULATOR")
+logger = logging.getLogger(__name__)
 
 class UnicornEmulator(object):
     '''

manticore/utils/log.py

@@ -6,75 +6,105 @@ class ContextFilter(logging.Filter):
     '''
     This is a filter which injects contextual information into the log.
     '''
+    def summarized_name(self, name):
+        '''
+        Produce a summarized record name
+          i.e. manticore.core.executor -> m.c.executor
+        '''
+        components = name.split('.')
+        prefix = '.'.join(c[0] for c in components[:-1])
+        return '{}.{}'.format(prefix, components[-1])
+
     def filter(self, record):
         if hasattr(self, 'stateid') and isinstance(self.stateid, int):
             record.stateid = '[%d]' % self.stateid
         else:
             record.stateid = ''
+
+        record.name = self.summarized_name(record.name)
         return True
 
-loggers = ['MANTICORE',
-           'VISITOR',
-           'EXECUTOR',
-           'CPU',
-           'REGISTERS',
-           'SMT',
-           'MEMORY',
-           'PLATFORM']
 manticore_verbosity = 0
+all_loggers = []
 
 def init_logging():
+    global all_loggers
+    all_loggers = logging.getLogger().manager.loggerDict.keys()
+        
     ctxfilter = ContextFilter()
     logfmt = ("%(asctime)s: [%(process)d]%(stateid)s %(name)s:%(levelname)s:"
               " %(message)s")
     logging.basicConfig(format=logfmt, stream=sys.stdout, level=logging.ERROR)
-    for l in loggers:
-        logging.getLogger(l).setLevel(logging.WARNING)
-        logging.getLogger(l).addFilter(ctxfilter)
-        logging.getLogger(l).setState = types.MethodType(loggerSetState,
-                                                         logging.getLogger(l))
+    for name in logging.getLogger().manager.loggerDict.keys():
+        logger = logging.getLogger(name)
+        if not name.startswith('manticore'):
+            next
+        logger.setLevel(logging.WARNING)
+        logger.addFilter(ctxfilter)
+        logger.setState = types.MethodType(loggerSetState, logger)
 
 def loggerSetState(logger, stateid):
     logger.filters[0].stateid = stateid
 
 def set_verbosity(setting):
-    global manticore_verbosity
-    zero = map(lambda x: (x, logging.WARNING), loggers)
+    global manticore_verbosity, all_loggers
+    zero = map(lambda x: (x, logging.WARNING), all_loggers)
     levels = [
         # 0
         zero,
         # 1
         [
-            ('MANTICORE', logging.INFO)
+            ('manticore.manticore', logging.INFO)
         ],
         # 2 (-v)
         [
-            ('EXECUTOR', logging.INFO),
-            ('PLATFORM', logging.DEBUG)
+            ('manticore.core.executor', logging.INFO),
+            ('manticore.platforms.*', logging.DEBUG)
         ],
         # 3 (-vv)
         [
-            ('CPU', logging.DEBUG)
+            ('manticore.core.cpu.*', logging.DEBUG)
         ],
         # 4 (-vvv)
         [
-            ('MEMORY', logging.DEBUG),
-            ('CPU', logging.DEBUG),
-            ('REGISTERS', logging.DEBUG)
+            ('manticore.core.memory', logging.DEBUG),
+            ('manticore.core.cpu.*', logging.DEBUG),
+            ('manticore.core.cpu.*.registers', logging.DEBUG)
         ],
         # 5 (-vvvv)
         [
-            ('MANTICORE', logging.DEBUG),
-            ('SMT', logging.DEBUG)
+            ('manticore.manticore', logging.DEBUG),
+            ('manticore.core.smtlib', logging.DEBUG),
+            ('manticore.core.smtlib.*', logging.DEBUG)
          ]
     ]
 
+    def match(name, pattern):
+        '''
+        Pseudo globbing that only supports full fields. 'a.*.d' matches 'a.b.d'
+        but not 'a.b.c.d'.
+        '''
+        name_l, pattern_l = name.split('.'), pattern.split('.')
+        if len(name_l) != len(pattern_l):
+            return False
+        for name_f, pattern_f in zip(name_l, pattern_l):
+            if pattern_f == '*':
+                continue
+            if name_f != pattern_f:
+                return False
+        return True
+
+    def glob(lst, expression):
+        return filter(lambda name: match(name, expression), lst)
+
     # Takes a value and ensures it's in a certain range
     def clamp(val, minimum, maximum):
         return sorted((minimum, val, maximum))[1]
 
     clamped = clamp(setting, 0, len(levels) - 1)
     for level in range(clamped + 1):
-        for log_type, log_level in levels[level]:
-            logging.getLogger(log_type).setLevel(log_level)
+        for pattern, log_level in levels[level]:
+            for logger_name in glob(all_loggers, pattern):
+                logger = logging.getLogger(logger_name)
+                logger.setLevel(log_level)
     manticore_verbosity = clamped