From 4f914e4eee760b21a7d3442de3785a6cca2984ef Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 1 Nov 2017 17:56:54 -0400 Subject: [PATCH] Fixes to stream formatting of doubles, they weren't being streamed before. Implemented the chk versions of printf and such, so that they all route through the logging interface as well. Implemented the concretization APIs. --- bin/deepstate/common.py | 69 +++++++++++------ bin/deepstate/main_angr.py | 31 +++++++- bin/deepstate/main_manticore.py | 19 +++++ examples/StreamingAndFormatting.cpp | 32 ++++---- src/lib/DeepState.c | 15 ++-- src/lib/Log.c | 110 ++++++++++++++++++++++------ src/lib/Stream.c | 16 ++-- 7 files changed, 216 insertions(+), 76 deletions(-) diff --git a/bin/deepstate/common.py b/bin/deepstate/common.py index dba1898..edfc3f6 100644 --- a/bin/deepstate/common.py +++ b/bin/deepstate/common.py @@ -74,6 +74,9 @@ class DeepState(object): def read_uint8_t(self, ea, concretize=True, constrain=False): raise NotImplementedError("Must be implemented by engine.") + def write_uint8_t(self, ea, val): + raise NotImplementedError("Must be implemented by engine.") + def concretize(self, val, constrain=False): raise NotImplementedError("Must be implemented by engine.") @@ -86,36 +89,27 @@ class DeepState(object): def add_constraint(self, expr): raise NotImplementedError("Must be implemented by engine.") - def read_c_string(self, ea, concretize=True): + def read_c_string(self, ea, concretize=True, constrain=False): """Read a NUL-terminated string from `ea`.""" assert isinstance(ea, (int, long)) chars = [] i = 0 while True: - b, ea = self.read_uint8_t(ea, concretize=concretize) + b, ea = self.read_uint8_t(ea, concretize=concretize, constrain=constrain) if self.is_symbolic(b): - concrete_b = self.concretize_min(b) # Find the NUL byte sooner. - if not concrete_b: - break - - if concretize: - chars.append(chr(concrete_b)) - else: - chars.append(b) - - continue - - # Concretize if it's not symbolic; we might have a concrete bitvector. - b = self.concretize(b) - if not b: - break - + b_maybe_nul = self.concretize_min(b) + if not b_maybe_nul: + break # Stop at the first possible NUL byte. else: - chars.append(chr(b)) + # Concretize if it's not symbolic; we might have a concrete bitvector. + b = self.concretize(b) + if not b: + break + chars.append(b) next_ea = ea + len(chars) + 1 if concretize: - return "".join(chars), next_ea + return "".join(chr(b) for b in chars), next_ea else: return chars, next_ea @@ -262,14 +256,45 @@ class DeepState(object): return 1 def api_assume(self, arg): - """Implements the `DeepState_Assume` API function, which injects a constraint - into the solver.""" + """Implements the `DeepState_Assume` API function, which injects a + constraint into the solver.""" constraint = arg != 0 if not self.add_constraint(constraint): self.log_message(LOG_LEVEL_FATAL, "Failed to add assumption {}".format(constraint)) self.abandon_test() + def api_concretize_data(self, begin_ea, end_ea): + """Implements the `Deeptate_ConcretizeData` API function, which lets the + programmer concretize some data in the exclusive range + `[begin_ea, end_ea)`.""" + begin_ea = self.concretize(begin_ea, constrain=True) + end_ea = self.concretize(end_ea, constrain=True) + if end_ea < begin_ea: + self.log_message( + LOG_LEVEL_FATAL, + "Invalid range [{:x}, {:x}) to McTest_Concretize".format( + begin_ea, end_ea)) + self.abandon_test() + + for i in xrange(end_ea - begin_ea): + val, _ = self.read_uint8_t(begin_ea + i, concretize=True, constrain=True) + _ = self.write_uint8_t(begin_ea + i, val) + + return begin_ea + + def api_concretize_cstr(self, begin_ea): + """Implements the `Deeptate_ConcretizeCStr` API function, which lets the + programmer concretize a NUL-terminated string starting at `begin_ea`.""" + begin_ea = self.concretize(begin_ea, constrain=True) + str_bytes, end_ea = self.read_c_string(begin_ea, concretize=False) + next_ea = begin_ea + for i, b in enumerate(str_bytes): + b = self.concretize_min(b, constrain=True) + next_ea = self.write_uint8_t(begin_ea + i, b) + self.write_uint8_t(next_ea, 0) + return begin_ea + def api_pass(self): """Implements the `DeepState_Pass` API function, which marks this test as having passed, and stops further execution.""" diff --git a/bin/deepstate/main_angr.py b/bin/deepstate/main_angr.py index def913a..212b3fa 100644 --- a/bin/deepstate/main_angr.py +++ b/bin/deepstate/main_angr.py @@ -76,6 +76,10 @@ class AngrTest(DeepState): val = ord(val) return val, ea + 1 + def write_uint8_t(self, ea, val): + self.state.mem[ea].uint8_t = val + return ea + 1 + def concretize(self, val, constrain=False): if isinstance(val, (int, long)): return val @@ -173,6 +177,21 @@ class SoftFail(angr.SimProcedure): AngrTest(procedure=self).api_soft_fail() +class ConcretizeData(angr.SimProcedure): + """Implements the `Deeptate_ConcretizeData` API function, which lets the + programmer concretize some data in the exclusive range + `[begin_ea, end_ea)`.""" + def run(self, begin_ea, end_ea): + return AngrTest(procedure=self).api_concretize_data(begin_ea, end_ea) + + +class ConcretizeCStr(angr.SimProcedure): + """Implements the `Deeptate_ConcretizeCStr` API function, which lets the + programmer concretize a NUL-terminated string starting at `begin_ea`.""" + def run(self, begin_ea): + return AngrTest(procedure=self).api_concretize_cstr(begin_ea) + + class StreamInt(angr.SimProcedure): """Implements _DeepState_StreamInt, which gives us an integer to stream, and the format to use for streaming.""" @@ -237,8 +256,9 @@ def do_run_test(project, test, apis, run_state): for state in test_manager.deadended: AngrTest(state=state).report() - for state in test_manager.errored: - print "ErrorL", state.error + for error in test_manager.errored: + print "Error", error.error + error.debug() def run_test(project, test, apis, run_state): """Symbolically executes a single test function.""" @@ -269,7 +289,10 @@ def main(): translation_cache=True, support_selfmodifying_code=False, auto_load_libs=True, - exclude_sim_procedures_list=['printf', 'fprintf']) + exclude_sim_procedures_list=['printf', '__printf_chk', + 'vprintf', '__vprintf_chk', + 'fprintf', '__fprintf_chk', + 'vfprintf', '__vfprintf_chk']) entry_state = project.factory.entry_state( add_options={angr.options.ZERO_FILL_UNCONSTRAINED_MEMORY, @@ -296,6 +319,8 @@ def main(): # Hook various functions. hook_function(project, apis['IsSymbolicUInt'], IsSymbolicUInt) + hook_function(project, apis['ConcretizeData'], ConcretizeData) + hook_function(project, apis['ConcretizeCStr'], ConcretizeCStr) hook_function(project, apis['Assume'], Assume) hook_function(project, apis['Pass'], Pass) hook_function(project, apis['Fail'], Fail) diff --git a/bin/deepstate/main_manticore.py b/bin/deepstate/main_manticore.py index dfaad68..4043b34 100644 --- a/bin/deepstate/main_manticore.py +++ b/bin/deepstate/main_manticore.py @@ -74,6 +74,10 @@ class MCoreTest(DeepState): val = ord(val) return val, ea + 1 + def write_uint8_t(self, ea, val): + self.state.cpu.write_int(ea, val, size=8) + return ea + 1 + def concretize(self, val, constrain=False): if isinstance(val, (int, long)): return val @@ -191,6 +195,19 @@ def hook_SoftFail(state): MCoreTest(state).api_soft_fail() +def hook_ConcretizeData(state, begin_ea, end_ea): + """Implements the `Deeptate_ConcretizeData` API function, which lets the + programmer concretize some data in the exclusive range + `[begin_ea, end_ea)`.""" + return MCoreTest(state).api_concretize_data(begin_ea, end_ea) + + +def hook_ConcretizeCStr(state, begin_ea): + """Implements the `Deeptate_ConcretizeCStr` API function, which lets the + programmer concretize a NUL-terminated string starting at `begin_ea`.""" + return MCoreTest(state).api_concretize_cstr(begin_ea) + + def hook_Log(state, level, ea): """Implements DeepState_Log, which lets Manticore intercept and handle the printing of log messages from the simulated tests.""" @@ -225,6 +242,8 @@ def do_run_test(state, apis, test): make_symbolic_input(state, apis['InputBegin'], apis['InputEnd']) m.add_hook(apis['IsSymbolicUInt'], hook(hook_IsSymbolicUInt)) + m.add_hook(apis['ConcretizeData'], hook(hook_ConcretizeData)) + m.add_hook(apis['ConcretizeCStr'], hook(hook_ConcretizeCStr)) m.add_hook(apis['Assume'], hook(hook_Assume)) m.add_hook(apis['Pass'], hook(hook_Pass)) m.add_hook(apis['Fail'], hook(hook_Fail)) diff --git a/examples/StreamingAndFormatting.cpp b/examples/StreamingAndFormatting.cpp index 960aa70..f12cbc6 100644 --- a/examples/StreamingAndFormatting.cpp +++ b/examples/StreamingAndFormatting.cpp @@ -16,22 +16,22 @@ #include -TEST(Streaming, BasicLevels) { - LOG(DEBUG) << "This is a debug message"; - LOG(INFO) << "This is an info message"; - LOG(WARNING) << "This is a warning message"; - LOG(ERROR) << "This is a error message"; - LOG(INFO) << "This is a info message again"; - ASSERT(true) << "This should not be printed."; -} - -TEST(Streaming, BasicTypes) { - LOG(INFO) << 'a'; - LOG(INFO) << 1; - LOG(INFO) << 1.0; - LOG(INFO) << "string"; - LOG(INFO) << nullptr; -} +//TEST(Streaming, BasicLevels) { +// LOG(DEBUG) << "This is a debug message"; +// LOG(INFO) << "This is an info message"; +// LOG(WARNING) << "This is a warning message"; +// LOG(ERROR) << "This is a error message"; +// LOG(INFO) << "This is a info message again"; +// ASSERT(true) << "This should not be printed."; +//} +// +//TEST(Streaming, BasicTypes) { +// LOG(INFO) << 'a'; +// LOG(INFO) << 1; +// LOG(INFO) << 1.0; +// LOG(INFO) << "string"; +// LOG(INFO) << nullptr; +//} TEST(Formatting, OverridePrintf) { printf("hello string=%s hex_lower=%x hex_upper=%X octal=%o char=%c dec=%d" diff --git a/src/lib/DeepState.c b/src/lib/DeepState.c index 2d9bb72..90b12a3 100644 --- a/src/lib/DeepState.c +++ b/src/lib/DeepState.c @@ -186,14 +186,17 @@ int DeepState_IsSymbolicUInt(uint32_t x) { } /* Defined in Stream.c */ -extern void _DeepState_StreamInt(enum DeepState_LogLevel level, const char *format, - const char *unpack, uint64_t *val); +extern void _DeepState_StreamInt(enum DeepState_LogLevel level, + const char *format, + const char *unpack, uint64_t *val); -extern void _DeepState_StreamFloat(enum DeepState_LogLevel level, const char *format, - const char *unpack, double *val); +extern void _DeepState_StreamFloat(enum DeepState_LogLevel level, + const char *format, + const char *unpack, double *val); -extern void _DeepState_StreamString(enum DeepState_LogLevel level, const char *format, - const char *str); +extern void _DeepState_StreamString(enum DeepState_LogLevel level, + const char *format, + const char *str); /* A DeepState-specific symbol that is needed for hooking. */ struct DeepState_IndexEntry { diff --git a/src/lib/Log.c b/src/lib/Log.c index 70d9820..6ba7403 100644 --- a/src/lib/Log.c +++ b/src/lib/Log.c @@ -14,12 +14,31 @@ * limitations under the License. */ +/* Helps to avoid conflicting declaration types of `__printf_chk`. */ +#define printf printf_foo +#define vprintf vprintf_foo +#define fprintf fprintf_foo +#define vfprintf vfprintf_foo +#define __printf_chk __printf_chk_foo +#define __vprintf_chk __vprintf_chk_foo +#define __fprintf_chk __fprintf_chk_foo +#define __vfprintf_chk __vfprintf_chk_foo + #include #include #include #include "deepstate/DeepState.h" +#undef printf +#undef vprintf +#undef fprintf +#undef vfprintf +#undef __printf_chk +#undef __vprintf_chk +#undef __fprintf_chk +#undef __vfprintf_chk + DEEPSTATE_BEGIN_EXTERN_C /* Returns a printable string version of the log level. */ @@ -47,9 +66,10 @@ enum { char DeepState_LogBuf[DeepState_LogBufSize + 1] = {}; /* Log a C string. */ +DEEPSTATE_NOINLINE void DeepState_Log(enum DeepState_LogLevel level, const char *str) { memset(DeepState_LogBuf, 0, DeepState_LogBufSize); - snprintf(DeepState_LogBuf, DeepState_LogBufSize, "%s: %s", + snprintf(DeepState_LogBuf, DeepState_LogBufSize, "%s: %s\n", DeepState_LogLevelStr(level), str); fputs(DeepState_LogBuf, stderr); @@ -61,48 +81,90 @@ void DeepState_Log(enum DeepState_LogLevel level, const char *str) { } /* Log some formatted output. */ -void DeepState_LogFormat(enum DeepState_LogLevel level, const char *format, ...) { +DEEPSTATE_NOINLINE +void DeepState_LogVFormat(enum DeepState_LogLevel level, + const char *format, va_list args) { DeepState_LogStream(level); - va_list args; - va_start(args, format); DeepState_StreamVFormat(level, format, args); - va_end(args); DeepState_LogStream(level); } /* Log some formatted output. */ -void DeepState_LogVFormat(enum DeepState_LogLevel level, - const char *format, va_list args) { - DeepState_LogStream(level); - DeepState_StreamVFormat(level, format, args); - DeepState_LogStream(level); +DEEPSTATE_NOINLINE +void DeepState_LogFormat(enum DeepState_LogLevel level, + const char *format, ...) { + va_list args; + va_start(args, format); + DeepState_LogVFormat(level, format, args); + va_end(args); } /* Override libc! */ +DEEPSTATE_NOINLINE int printf(const char *format, ...) { - DeepState_LogStream(DeepState_LogInfo); va_list args; va_start(args, format); - DeepState_StreamVFormat(DeepState_LogInfo, format, args); + DeepState_LogVFormat(DeepState_LogInfo, format, args); va_end(args); - DeepState_LogStream(DeepState_LogInfo); return 0; } -int fprintf(FILE *file, const char *format, ...) { - enum DeepState_LogLevel level = DeepState_LogInfo; - if (stderr == file) { - level = DeepState_LogDebug; - } else if (stdout != file) { - return 0; /* TODO(pag): This is probably evil. */ - } - - DeepState_LogStream(level); +DEEPSTATE_NOINLINE +int __printf_chk(int flag, const char *format, ...) { va_list args; va_start(args, format); - DeepState_StreamVFormat(level, format, args); + DeepState_LogVFormat(DeepState_LogInfo, format, args); va_end(args); - DeepState_LogStream(level); + return 0; +} + +DEEPSTATE_NOINLINE +int vprintf(const char *format, va_list args) { + DeepState_LogVFormat(DeepState_LogInfo, format, args); + return 0; +} + +DEEPSTATE_NOINLINE +int __vprintf_chk(int flag, const char *format, va_list args) { + DeepState_LogVFormat(DeepState_LogInfo, format, args); + return 0; +} + +DEEPSTATE_NOINLINE +int vfprintf(FILE *file, const char *format, va_list args) { + if (stderr == file) { + DeepState_LogVFormat(DeepState_LogDebug, format, args); + } else if (stdout == file) { + DeepState_LogVFormat(DeepState_LogInfo, format, args); + } else { + DeepState_LogStream(DeepState_LogWarning); + DeepState_Log(DeepState_LogWarning, + "Ignorning vfprintf with non-stdout/stderr stream."); + } + return 0; +} + +DEEPSTATE_NOINLINE +int fprintf(FILE *file, const char *format, ...) { + va_list args; + va_start(args, format); + vfprintf(file, format, args); + va_end(args); + return 0; +} + +DEEPSTATE_NOINLINE +int __fprintf_chk(int flag, FILE *file, const char *format, ...) { + va_list args; + va_start(args, format); + vfprintf(file, format, args); + va_end(args); + return 0; +} + +DEEPSTATE_NOINLINE +int __vfprintf_chk(int flag, FILE *file, const char *format, va_list args) { + vfprintf(file, format, args); return 0; } diff --git a/src/lib/Stream.c b/src/lib/Stream.c index d379fd0..fb3877a 100644 --- a/src/lib/Stream.c +++ b/src/lib/Stream.c @@ -255,8 +255,7 @@ void DeepState_StreamDouble(enum DeepState_LogLevel level, double val) { void DeepState_LogStream(enum DeepState_LogLevel level) { struct DeepState_Stream *stream = &(DeepState_Streams[level]); if (stream->size) { - stream->message[stream->size] = '\n'; - stream->message[stream->size + 1] = '\0'; + stream->message[stream->size] = '\0'; stream->message[DeepState_StreamSize] = '\0'; DeepState_Log(level, stream->message); memset(stream->message, 0, DeepState_StreamSize); @@ -501,6 +500,8 @@ get_length_char: stream->value.as_fp64 = va_arg(args, double); } DeepState_StreamUnpack(stream, 'd'); + _DeepState_StreamFloat(level, format_buf, stream->unpack, + &(stream->value.as_fp64)); break; case 's': @@ -527,14 +528,19 @@ void DeepState_StreamVFormat(enum DeepState_LogLevel level, const char *format_, va_list args) { char *begin = NULL; char *end = NULL; - char *format = DeepState_Format; + char *format = &(DeepState_Format[0]); int i = 0; char ch = '\0'; char next_ch = '\0'; + size_t len = strlen(format_); - strncpy(format, format_, DeepState_StreamSize); - format[DeepState_StreamSize] = '\0'; + if (len >= DeepState_StreamSize) { + DeepState_Abandon("Format string is too long."); + } + /* Concretize the string format. */ + memcpy(format, format_, len); + format[len] = '\0'; DeepState_ConcretizeCStr(format); for (i = 0; '\0' != (ch = format[i]); ) {