Run linux examples in Travis (#668)

* Update makefile; add a list target for testing

* simplify nostdlib example

* Make sendmail example return success

* Add tests to run all examples

* Add some targets to exclude

* Run example scripts; temporarily add a workspace accsesor to mcore

* Optionally read end of main from argv

* Make concolic test more robust

* Clean up Makefile

* Be better with phony targets

* Add run_simple and state_control tests

* verbosity++

* Make sure we fail when we intend to

* Simplify travis_test.sh

* Remove multi_arch_sym

examples/linux/Makefile

@@ -1,55 +1,33 @@
 CC=gcc
 CFLAGS=-O3 -static
+NOSTDLIBFLAGS=-fno-builtin -static -nostdlib -fomit-frame-pointer
 
-all: CFLAGS=-O3 -static 
+EXAMPLES=basic sindex  strncmp  arguments ibranch sendmail crackme indexhell  helloworld simple_copy simpleassert
-all: NOSTDLIBFLAGS=-m32 -fno-builtin -static -nostdlib  -fomit-frame-pointer
+OTHER_EXAMPLES=nostdlib
-all: nostdlib  basic  sindex  strncmp  arguments ibranch sendmail crackme indexhell baby-re helloworld simpleassert
+
+all: $(EXAMPLES) $(OTHER_EXAMPLES) 
 
 arm: CC=arm-linux-gnueabi-gcc
-arm: basic  sindex  strncmp  arguments ibranch sendmail crackme indexhell helloworld simple_copy simpleassert
+arm: $(EXAMPLES)
+
+.PHONY: list clean
+list:
+	@echo $(EXAMPLES)
 
 clean:
-		rm -rf nostdlib  basic  sindex strncmp arguments sendmail server ibranch crackme indexhell crackme.c simple_copy helloworld nostdlib32 nostdlib64
+	rm -rf $(EXAMPLES) $(OTHER_EXAMPLES) crackme.c
+
+% : %.c
+	$(CC) $(CFLAGS) $< -o $@
 
 nostdlib: nostdlib.c 
-		$(CC) -m32 -fno-builtin -static -nostdlib  -fomit-frame-pointer  nostdlib.c  -o nostdlib32
+	$(CC) -m32 $(NOSTDLIBFLAGS) $< -o $@
-		$(CC) -m32 -fno-builtin -static -nostdlib  -fomit-frame-pointer  nostdlib.c  -o nostdlib64
-
-helloworld: helloworld.c
-		$(CC) $(CFLAGS) $< -static -o $@
-
-simple_copy: simple_copy.c
-		$(CC) $(CFLAGS) simple_copy.c -static -o simple_copy
-basic: basic.c
-		$(CC) $(CFLAGS) basic.c -static -o basic
 
+# simpleassert needs -O0
 simpleassert: simpleassert.c
-		$(CC) $(CFLAGS) -O0 simpleassert.c -static -o simpleassert
+	$(CC) $(CFLAGS) -O0 $< -o $@
 
-sindex: sindex.c
+# crackme needs to be generated
-		$(CC) $(CFLAGS) sindex.c -o sindex
+crackme.c: crackme.py
+	python crackme.py > $@
 
-ibranch: ibranch.c
-		$(CC) $(CFLAGS) ibranch.c -o ibranch
-
-strncmp: strncmp.c
-		$(CC) $(CFLAGS) strncmp.c -o strncmp
-
-arguments: arguments.c
-		$(CC) $(CFLAGS) arguments.c  -o arguments
-
-sendmail: sendmail.c
-		gcc -static sendmail.c -o sendmail
-
-server: server.c
-		gcc -static server.c -o server
-
-crackme: crackme.py 
-	python crackme.py  >crackme.c
-	gcc -static -Os crackme.c -o crackme
-
-indexhell: indexhell.c
-	gcc -static indexhell.c -o indexhell
-
-baby-re: baby-re.c
-	$(CC) $(CFLAGS) -o $@ $< -Wno-unused-result

examples/linux/nostdlib.c

@@ -19,36 +19,34 @@
         arg 6           %ebp         call-saved
 */
 static inline
-int syscall(int syscall_number, ... )  {
+int syscall(int syscall_number, int arg1, int arg2, int arg3)  {
     int ret;
     asm volatile (
         "pushl %%ebp\n\t"
         "movl %1, %%eax\n\t"
-        "movl %2, %%ebx\n\t"
+        "movl %2, %%eax\n\t"
-        "movl %3, %%ecx\n\t"
+        "movl %3, %%ebx\n\t"
-        "movl %4, %%edx\n\t"
+        "movl %4, %%ecx\n\t"
-        "movl %5, %%edi\n\t"
+        //"movl %4, %%edx\n\t"
-        "movl %6, %%esi\n\t"
-        "movl %7, %%ebp\n\t"
         "int $0x80\n\t"
         "popl %%ebp\n\t"
         : "=a"(ret)
-        : "g"(syscall_number), "g"(*(&syscall_number+1)), "g"(*(&syscall_number+2)), "g"(*(&syscall_number+3)), "g"(*(&syscall_number+4)), "g"(*(&syscall_number+5)), "g"(*(&syscall_number+6))
+        : "g"(syscall_number), "g"(arg1), "g"(arg2), "g"(arg3)
         : "%ebx", "%ecx", "%edx", "%esi", "%edi"
     );
 return ret;
 }
 
 int write(int fd, void* buffer, unsigned int size){
-    return syscall(4, fd, buffer, size,0,0,0);
+    return syscall(4, fd, (int) buffer, size);
 }
 
 int read(int fd, void* buffer, unsigned int size){
-    return syscall(3, fd, buffer, size,0,0,0);
+    return syscall(3, fd, (int) buffer, size);
 }
 
 int exit(int errorlevel){
-    return syscall(1, errorlevel,0,0,0,0,0);
+    return syscall(1, errorlevel,0,0);
 }
 
 void _start(){

examples/linux/sendmail.c

@@ -62,4 +62,5 @@ main(int argc, char argv[]){
 char buffer[200];
 read(0,buffer,200);
 copy_it(buffer, 200);
+return 0;
 }

examples/script/concolic.py

@@ -13,6 +13,7 @@ Bugs
 
 '''
 
+import sys
 import Queue
 import struct
 import itertools
@@ -27,7 +28,7 @@ import copy
 from manticore.core.smtlib.expression import *
 
 prog = '../linux/simpleassert'
-endd = 0x400ae9
+main_end = 0x400ae9
 VERBOSITY = 0
 
 def _partition(pred, iterable):
@@ -163,7 +164,7 @@ def symbolic_run_get_cons(trace):
     m2.verbosity(VERBOSITY)
     m2.register_plugin(f)
 
-    @m2.hook(endd)
+    @m2.hook(main_end)
     def x(s):
         with m2.locked_context() as ctx:
             readdata = []
@@ -250,6 +251,12 @@ def concrete_input_to_constraints(ci, prev=None):
 
 
 def main():
+    global main_end
+
+    # Read the address of main's `ret` from cmdline if we're passed it. Used for testing.
+    if len(sys.argv) > 1:
+        main_end = int(sys.argv[1], 0)
+        log("Got end of main: {:x}".format(main_end))
 
     q = Queue.Queue()
 

examples/script/multi_arch_sym.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-from manticore import Manticore
-
-'''
-Minimal example demonstrating setting execution hooks, the ability to target
-multiple target architectures, and symbolicating memory.
-'''
-
-if __name__ == '__main__':
-
-    if len(sys.argv) < 2:
-        print "Usage: {} [binary] [arguments]".format(sys.argv[0])
-        sys.exit(1)
-
-    # Create a new Manticore object
-    m = Manticore(sys.argv[1], sys.argv[2:])
-    m.verbosity = 2
-
-    if m.arch == 'arm':
-        target = (0x1082c, 'R4')
-    else:
-        target = (0x400a83, 'EBX')
-
-    @m.hook(target[0])
-    def entered_func(state):
-        '''
-        For ARM, Make R4 symbolic at 0x1082c, as r4 is used in a branch right
-        after.
-        '''
-        sym_var = state.new_symbolic_value(32, label='from_callback')
-        state.cpu.write_register(target[1], sym_var)
-
-    m.run()
-

manticore/manticore.py

@@ -662,6 +662,10 @@ class Manticore(Eventful):
     def coverage_file(self):
         return self._coverage_file
 
+    @property
+    def workspace(self):
+         return self._output.store.uri
+
     @coverage_file.setter
     def coverage_file(self, path):
         assert not self.running, "Can't set coverage file if Manticore is running."

scripts/travis_test.sh

@@ -1,13 +1,79 @@
 #!/bin/bash
+
 RV=0
-cd examples/linux
+
-if make; then
+set -o errexit
-    echo "Successfully built Linux examples"
+set -o pipefail
-else
+
-    echo "Failed to build Linux examples"
+# Run all examples; this assumes PWD is examples/script
-    RV=1
+run_examples() {
+    # concolic assumes presence of ../linux/simpleassert
+    echo "Running concolic.py..."
+    HW=../linux/helloworld
+    SA=../linux/simpleassert
+    END_OF_MAIN=$(objdump -d $SA|awk -v RS= '/^[[:xdigit:]].*<main>/'|grep ret|tr  -d ' ' | awk -F: '{print "0x" $1}')
+    python ./concolic.py $END_OF_MAIN
+    if [ $? -ne 0 ]; then
+        return 1
     fi
-cd ../..
+
+    echo "Running count_instructions.py..."
+    python ./count_instructions.py $HW |grep -q Executed
+    if [ $? -ne 0 ]; then
+        return 1
+    fi
+
+    echo "Running introduce_symbolic_bytes.py..."
+    gcc -static -g src/state_explore.c -o state_explore
+    ADDRESS=0x$(objdump -S state_explore | grep -A 1 '((value & 0xff) != 0)' |
+            tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:)
+    python ./introduce_symbolic_bytes.py state_explore $ADDRESS
+    if [ $? -ne 0 ]; then
+        return 1
+    fi
+
+    echo "Running run_simple.py..."
+    gcc -x c -static -o hello - <<-EOF
+    #include <stdio.h>
+    int main(){return 0;}
+	EOF
+    python ./run_simple.py hello
+    if [ $? -ne 0 ]; then
+        return 1
+    fi
+
+    echo "Running run_hook.py..."
+    MAIN_ADDR=$(nm $HW|grep 'T main' | awk '{print "0x"$1}')
+    python ./run_hook.py $HW $MAIN_ADDR
+    if [ $? -ne 0 ]; then
+        return 1
+    fi
+
+    echo "Running state_control.py..."
+    # Straight from the header of state_control.py
+    gcc -static -g src/state_explore.c -o state_explore
+    SE_ADDR=0x$(objdump -S state_explore | grep -A 1 'value == 0x41' |
+               tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:)
+    python ./state_control.py state_explore $SE_ADDR
+    if [ $? -ne 0 ]; then
+        return 1
+    fi
+
+    return 0
+}
+
+pushd examples/linux
+make
+for example in $(make list); do
+    ./$example < /dev/zero > /dev/null
+done
+echo Built and ran Linux examples
+popd
+
+pushd examples/script
+run_examples
+echo Ran example scripts
+popd
 
 coverage erase
 coverage run -m unittest discover tests/ 2>&1 >/dev/null | tee travis_tests.log
@@ -17,7 +83,7 @@ then
     echo "All functionality tests passed :)"
 else
     echo "Some functionality tests failed :("
-    RV=1
+    exit 2
 fi
 
 measure_cov() {
@@ -27,8 +93,7 @@ measure_cov() {
     if [ "${HAS_COV}" = "No data to report" ]
     then
         echo "    FAIL: No coverage for ${PYFILE}"
-        RV=1
+        return 1
-        return
     fi
     
     local COV_AMT=$(coverage report --include=${PYFILE} | tail -n1 | sed "s/.* \([0-9]*\)%/\1/g")
@@ -37,8 +102,9 @@ measure_cov() {
         echo "    PASS: coverage for ${PYFILE} at ${COV_AMT}%"
     else
         echo "    FAIL: coverage for ${PYFILE} at ${COV_AMT}%"
-        RV=1
+        return 1
     fi
+    return 0
 }
 
 #coverage report