From 1edee15e5f56001728057865de1981f2dcbfc3ff Mon Sep 17 00:00:00 2001 From: Yan Date: Mon, 25 Sep 2017 18:09:18 -0400 Subject: [PATCH] Pythonic logger naming (#511) * More generic logging * Clean up logger name gen * Refactor name summarization --- manticore/core/cpu/abstractcpu.py | 4 +- manticore/core/cpu/arm.py | 2 +- manticore/core/cpu/binja.py | 5 +- manticore/core/cpu/x86.py | 3 +- manticore/core/executor.py | 2 +- manticore/core/memory.py | 2 +- manticore/core/smtlib/__init__.py | 2 +- manticore/core/smtlib/constraints.py | 3 +- manticore/core/smtlib/solver.py | 2 +- manticore/core/smtlib/visitors.py | 2 +- manticore/core/state.py | 2 +- manticore/core/workspace.py | 2 +- manticore/manticore.py | 3 +- manticore/platforms/decree.py | 2 +- manticore/platforms/linux.py | 2 +- manticore/platforms/windows.py | 2 +- manticore/utils/emulate.py | 2 +- manticore/utils/log.py | 82 +++++++++++++++++++--------- 18 files changed, 77 insertions(+), 47 deletions(-) diff --git a/manticore/core/cpu/abstractcpu.py b/manticore/core/cpu/abstractcpu.py index aaf2bf9..fbfb402 100644 --- a/manticore/core/cpu/abstractcpu.py +++ b/manticore/core/cpu/abstractcpu.py @@ -18,8 +18,8 @@ from ...utils.helpers import issymbolic from ...utils.emulate import UnicornEmulator from ...utils.event import Eventful -logger = logging.getLogger("CPU") -register_logger = logging.getLogger("REGISTERS") +logger = logging.getLogger(__name__) +register_logger = logging.getLogger('{}.registers'.format(__name__)) ################################################################################### #Exceptions diff --git a/manticore/core/cpu/arm.py b/manticore/core/cpu/arm.py index 37a0632..ca14c4f 100644 --- a/manticore/core/cpu/arm.py +++ b/manticore/core/cpu/arm.py @@ -18,7 +18,7 @@ from .register import Register from ..smtlib import Operators, Expression, BitVecConstant from ...utils.helpers import issymbolic -logger = logging.getLogger("CPU") +logger = logging.getLogger(__name__) # map different instructions to a single impl here OP_NAME_MAP = { diff --git a/manticore/core/cpu/binja.py b/manticore/core/cpu/binja.py index 1f5226e..216626c 100644 --- a/manticore/core/cpu/binja.py +++ b/manticore/core/cpu/binja.py @@ -16,9 +16,8 @@ from ...core.cpu.disasm import BinjaILDisasm from ..smtlib import Operators, BitVecConstant, operator from ...utils.helpers import issymbolic -logger = logging.getLogger("CPU") -register_logger = logging.getLogger("REGISTERS") - +logger = logging.getLogger(__name__) +register_logger = logging.getLogger('{}.registers'.format(__name__)) class BinjaRegisterFile(RegisterFile): diff --git a/manticore/core/cpu/x86.py b/manticore/core/cpu/x86.py index 7fe6366..35d0adc 100644 --- a/manticore/core/cpu/x86.py +++ b/manticore/core/cpu/x86.py @@ -16,8 +16,7 @@ from ..smtlib import Operators, BitVec, Bool, BitVecConstant, operator, visitors from ..memory import MemoryException from ...utils.helpers import issymbolic -logger = logging.getLogger("CPU") - +logger = logging.getLogger(__name__) OP_NAME_MAP = { 'JNE': 'JNZ', diff --git a/manticore/core/executor.py b/manticore/core/executor.py index d4dbdb2..12ba236 100644 --- a/manticore/core/executor.py +++ b/manticore/core/executor.py @@ -23,7 +23,7 @@ def mgr_init(): manager = SyncManager() manager.start(mgr_init) -logger = logging.getLogger("EXECUTOR") +logger = logging.getLogger(__name__) def sync(f): diff --git a/manticore/core/memory.py b/manticore/core/memory.py index 6d6e0ff..b947eb9 100644 --- a/manticore/core/memory.py +++ b/manticore/core/memory.py @@ -6,7 +6,7 @@ import logging from ..utils.mappings import _mmap, _munmap from ..utils.helpers import issymbolic -logger = logging.getLogger('MEMORY') +logger = logging.getLogger(__name__) class MemoryException(Exception): diff --git a/manticore/core/smtlib/__init__.py b/manticore/core/smtlib/__init__.py index 0d8eb32..e3f21b6 100644 --- a/manticore/core/smtlib/__init__.py +++ b/manticore/core/smtlib/__init__.py @@ -7,7 +7,7 @@ import math import logging -logger = logging.getLogger("SMT") +logger = logging.getLogger(__name__) ''' class OperationNotPermited(SolverException): diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 3bca41e..a325eef 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -1,7 +1,8 @@ from expression import BitVecVariable, BoolVariable, ArrayVariable, Array, Bool, BitVec, BitVecConstant, BoolConstant, ArrayProxy from visitors import GetDeclarations, TranslatorSmtlib, ArithmeticSimplifier, PrettyPrinter, pretty_print, translate_to_smtlib, get_depth, get_variables, arithmetic_simplifier import logging, weakref -logger = logging.getLogger('SMT') + +logger = logging.getLogger(__name__) class ConstraintSet(object): ''' Constraint Sets diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 483100f..a25cf37 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -26,7 +26,7 @@ from visitors import * from ...utils.helpers import issymbolic, memoized import collections -logger = logging.getLogger("SMT") +logger = logging.getLogger(__name__) class Z3NotFoundError(EnvironmentError): pass diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 472553d..ef169a1 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -1,7 +1,7 @@ from abc import ABCMeta, abstractmethod, abstractproperty from expression import * import logging -logger = logging.getLogger("VISITOR") +logger = logging.getLogger(__name__) class Visitor(object): diff --git a/manticore/core/state.py b/manticore/core/state.py index 43e7da3..900839d 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -12,7 +12,7 @@ from .cpu.abstractcpu import ConcretizeRegister from .memory import ConcretizeMemory, MemoryException from ..platforms.platform import * -logger = logging.getLogger("STATE") +logger = logging.getLogger(__name__) class StateException(Exception): ''' All state related exceptions ''' diff --git a/manticore/core/workspace.py b/manticore/core/workspace.py index 4ec95f2..a7484a0 100644 --- a/manticore/core/workspace.py +++ b/manticore/core/workspace.py @@ -16,7 +16,7 @@ from multiprocessing.managers import SyncManager from .smtlib import solver from .smtlib.solver import SolverException -logger = logging.getLogger('WORKSPACE') +logger = logging.getLogger(__name__) manager = SyncManager() manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) diff --git a/manticore/manticore.py b/manticore/manticore.py index b0024d1..106ff63 100644 --- a/manticore/manticore.py +++ b/manticore/manticore.py @@ -27,8 +27,9 @@ from .utils.nointerrupt import WithKeyboardInterruptAs import logging from .utils import log + +logger = logging.getLogger(__name__) log.init_logging() -logger = logging.getLogger('MANTICORE') def make_binja(program, disasm, argv, env, symbolic_files, concrete_start=''): def _check_disassembler_present(disasm): diff --git a/manticore/platforms/decree.py b/manticore/platforms/decree.py index 2005984..2236427 100644 --- a/manticore/platforms/decree.py +++ b/manticore/platforms/decree.py @@ -15,7 +15,7 @@ import StringIO import logging import random -logger = logging.getLogger("PLATFORM") +logger = logging.getLogger(__name__) class RestartSyscall(Exception): pass diff --git a/manticore/platforms/linux.py b/manticore/platforms/linux.py index abe93d6..bb6795c 100644 --- a/manticore/platforms/linux.py +++ b/manticore/platforms/linux.py @@ -21,7 +21,7 @@ from ..platforms.platform import Platform from ..utils.helpers import issymbolic, is_binja_disassembler from . import linux_syscalls -logger = logging.getLogger("PLATFORM") +logger = logging.getLogger(__name__) class RestartSyscall(Exception): pass diff --git a/manticore/platforms/windows.py b/manticore/platforms/windows.py index 4ce5c65..92fd128 100644 --- a/manticore/platforms/windows.py +++ b/manticore/platforms/windows.py @@ -17,7 +17,7 @@ import StringIO import logging import random from windows_syscalls import syscalls_num -logger = logging.getLogger("PLATFORM") +logger = logging.getLogger(__name__) class SyscallNotImplemented(TerminateState): diff --git a/manticore/utils/emulate.py b/manticore/utils/emulate.py index 7625950..9549072 100644 --- a/manticore/utils/emulate.py +++ b/manticore/utils/emulate.py @@ -15,7 +15,7 @@ from capstone import * from capstone.arm import * from capstone.x86 import * -logger = logging.getLogger("EMULATOR") +logger = logging.getLogger(__name__) class UnicornEmulator(object): ''' diff --git a/manticore/utils/log.py b/manticore/utils/log.py index 13d60f7..b2cd9c0 100644 --- a/manticore/utils/log.py +++ b/manticore/utils/log.py @@ -6,75 +6,105 @@ class ContextFilter(logging.Filter): ''' This is a filter which injects contextual information into the log. ''' + def summarized_name(self, name): + ''' + Produce a summarized record name + i.e. manticore.core.executor -> m.c.executor + ''' + components = name.split('.') + prefix = '.'.join(c[0] for c in components[:-1]) + return '{}.{}'.format(prefix, components[-1]) + def filter(self, record): if hasattr(self, 'stateid') and isinstance(self.stateid, int): record.stateid = '[%d]' % self.stateid else: record.stateid = '' + + record.name = self.summarized_name(record.name) return True -loggers = ['MANTICORE', - 'VISITOR', - 'EXECUTOR', - 'CPU', - 'REGISTERS', - 'SMT', - 'MEMORY', - 'PLATFORM'] manticore_verbosity = 0 +all_loggers = [] def init_logging(): + global all_loggers + all_loggers = logging.getLogger().manager.loggerDict.keys() + ctxfilter = ContextFilter() logfmt = ("%(asctime)s: [%(process)d]%(stateid)s %(name)s:%(levelname)s:" " %(message)s") logging.basicConfig(format=logfmt, stream=sys.stdout, level=logging.ERROR) - for l in loggers: - logging.getLogger(l).setLevel(logging.WARNING) - logging.getLogger(l).addFilter(ctxfilter) - logging.getLogger(l).setState = types.MethodType(loggerSetState, - logging.getLogger(l)) + for name in logging.getLogger().manager.loggerDict.keys(): + logger = logging.getLogger(name) + if not name.startswith('manticore'): + next + logger.setLevel(logging.WARNING) + logger.addFilter(ctxfilter) + logger.setState = types.MethodType(loggerSetState, logger) def loggerSetState(logger, stateid): logger.filters[0].stateid = stateid def set_verbosity(setting): - global manticore_verbosity - zero = map(lambda x: (x, logging.WARNING), loggers) + global manticore_verbosity, all_loggers + zero = map(lambda x: (x, logging.WARNING), all_loggers) levels = [ # 0 zero, # 1 [ - ('MANTICORE', logging.INFO) + ('manticore.manticore', logging.INFO) ], # 2 (-v) [ - ('EXECUTOR', logging.INFO), - ('PLATFORM', logging.DEBUG) + ('manticore.core.executor', logging.INFO), + ('manticore.platforms.*', logging.DEBUG) ], # 3 (-vv) [ - ('CPU', logging.DEBUG) + ('manticore.core.cpu.*', logging.DEBUG) ], # 4 (-vvv) [ - ('MEMORY', logging.DEBUG), - ('CPU', logging.DEBUG), - ('REGISTERS', logging.DEBUG) + ('manticore.core.memory', logging.DEBUG), + ('manticore.core.cpu.*', logging.DEBUG), + ('manticore.core.cpu.*.registers', logging.DEBUG) ], # 5 (-vvvv) [ - ('MANTICORE', logging.DEBUG), - ('SMT', logging.DEBUG) + ('manticore.manticore', logging.DEBUG), + ('manticore.core.smtlib', logging.DEBUG), + ('manticore.core.smtlib.*', logging.DEBUG) ] ] + def match(name, pattern): + ''' + Pseudo globbing that only supports full fields. 'a.*.d' matches 'a.b.d' + but not 'a.b.c.d'. + ''' + name_l, pattern_l = name.split('.'), pattern.split('.') + if len(name_l) != len(pattern_l): + return False + for name_f, pattern_f in zip(name_l, pattern_l): + if pattern_f == '*': + continue + if name_f != pattern_f: + return False + return True + + def glob(lst, expression): + return filter(lambda name: match(name, expression), lst) + # Takes a value and ensures it's in a certain range def clamp(val, minimum, maximum): return sorted((minimum, val, maximum))[1] clamped = clamp(setting, 0, len(levels) - 1) for level in range(clamped + 1): - for log_type, log_level in levels[level]: - logging.getLogger(log_type).setLevel(log_level) + for pattern, log_level in levels[level]: + for logger_name in glob(all_loggers, pattern): + logger = logging.getLogger(logger_name) + logger.setLevel(log_level) manticore_verbosity = clamped