Add tests for Crash, Klee, Lists examples; detect internal errors and exceptions

.travis.yml

@@ -17,6 +17,12 @@ install:
 - python setup.py install
 - cd ..
 env:
+- TASK=CRASH DEEPSTATE_CMD=deepstate-angr
+- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore
+- TASK=KLEE DEEPSTATE_CMD=deepstate-angr
+- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore
+- TASK=LISTS DEEPSTATE_CMD=deepstate-angr
+- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore
 - TASK=PRIMES DEEPSTATE_CMD=deepstate-angr
 - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore
 - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr

tests/logrun.py

@@ -19,5 +19,9 @@ def logrun(cmd, file, timeout):
     sys.stderr.write("\n")
     if p.poll() is None:
         return ("TIMEOUT", contents)
+    if "internal error" in contents:
+        return ("INTERNAL ERROR", contents)
+    if "Traceback (most recent call last)" in contents:
+        return ("EXCEPTION RAISED", contents)    
     return (p.returncode, contents)
         

tests/test_basic_functionality.py

@@ -10,10 +10,30 @@ class TestBasicFunctionality(TestCase):
         if deepstate is None:
             deepstate = "deepstate-angr" # default to angr in an environment without a defined command
 
+        if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH":
+            (r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
+                                        "deepstate.out", 1800)
+            self.assertEqual(r, 0)
+                
+            self.assertTrue("Passed: Crash_SegFault" in output)
+            foundCrashSave = False
+            for line in output.split("\n"):
+                if ("Saving input to" in line) and (".crash" in line):
+                    foundCrashSave = True
+            self.assertTrue(foundCrashSave)
+
+        if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE":
+            (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
+                                        "deepstate.out", 1800)
+            self.assertEqual(r, 0)            
+
+            self.assertTrue("zero" in output)
+            self.assertTrue("positive" in output)
+            self.assertTrue("negative" in output)
+            
         if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
             (r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
                                         "deepstate.out", 1800)
-            
             self.assertEqual(r, 0)
 
             self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
@@ -22,10 +42,17 @@ class TestBasicFunctionality(TestCase):
             self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
             self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
 
+        if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS":
+            (r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
+                                        "deepstate.out", 1800)
+            self.assertEqual(r, 0)            
+
+            self.assertTrue("Passed: Vector_DoubleReversal" in output)
+            self.assertFalse("Failed: Vector_DoubleReversal" in output)             
+
         if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
             (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
                                         "deepstate.out", 1800)
-            
             self.assertEqual(r, 0)
 
             self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
@@ -34,8 +61,12 @@ class TestBasicFunctionality(TestCase):
         if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
             (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
                                         "deepstate.out", 1800)
+            self.assertEqual(r, 0)
 
             self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
             self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
+            self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)            
             self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
+            self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)            
             self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
+