From eb150fc13c4ca3bdacbac1955bac3550f2a3711f Mon Sep 17 00:00:00 2001
From: Alex Groce <agroce@gmail.com>
Date: Fri, 13 Jul 2018 20:50:00 -0700
Subject: [PATCH] Add tests for Crash, Klee, Lists examples; detect internal
 errors and exceptions

---
 .travis.yml                       |  6 ++++++
 tests/logrun.py                   |  4 ++++
 tests/test_basic_functionality.py | 35 +++++++++++++++++++++++++++++--
 3 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 45b70c7..93732b0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,12 @@ install:
 - python setup.py install
 - cd ..
 env:
+- TASK=CRASH DEEPSTATE_CMD=deepstate-angr
+- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore
+- TASK=KLEE DEEPSTATE_CMD=deepstate-angr
+- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore
+- TASK=LISTS DEEPSTATE_CMD=deepstate-angr
+- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore
 - TASK=PRIMES DEEPSTATE_CMD=deepstate-angr
 - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore
 - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr
diff --git a/tests/logrun.py b/tests/logrun.py
index d60185c..12bafe5 100644
--- a/tests/logrun.py
+++ b/tests/logrun.py
@@ -19,5 +19,9 @@ def logrun(cmd, file, timeout):
     sys.stderr.write("\n")
     if p.poll() is None:
         return ("TIMEOUT", contents)
+    if "internal error" in contents:
+        return ("INTERNAL ERROR", contents)
+    if "Traceback (most recent call last)" in contents:
+        return ("EXCEPTION RAISED", contents)    
     return (p.returncode, contents)
         
diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py
index 42966fd..2f16943 100644
--- a/tests/test_basic_functionality.py
+++ b/tests/test_basic_functionality.py
@@ -10,10 +10,30 @@ class TestBasicFunctionality(TestCase):
         if deepstate is None:
             deepstate = "deepstate-angr" # default to angr in an environment without a defined command
 
+        if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH":
+            (r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
+                                        "deepstate.out", 1800)
+            self.assertEqual(r, 0)
+                
+            self.assertTrue("Passed: Crash_SegFault" in output)
+            foundCrashSave = False
+            for line in output.split("\n"):
+                if ("Saving input to" in line) and (".crash" in line):
+                    foundCrashSave = True
+            self.assertTrue(foundCrashSave)
+
+        if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE":
+            (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
+                                        "deepstate.out", 1800)
+            self.assertEqual(r, 0)            
+
+            self.assertTrue("zero" in output)
+            self.assertTrue("positive" in output)
+            self.assertTrue("negative" in output)
+            
         if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
             (r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
                                         "deepstate.out", 1800)
-            
             self.assertEqual(r, 0)
 
             self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
@@ -22,10 +42,17 @@ class TestBasicFunctionality(TestCase):
             self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
             self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
 
+        if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS":
+            (r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
+                                        "deepstate.out", 1800)
+            self.assertEqual(r, 0)            
+
+            self.assertTrue("Passed: Vector_DoubleReversal" in output)
+            self.assertFalse("Failed: Vector_DoubleReversal" in output)             
+
         if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
             (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
                                         "deepstate.out", 1800)
-            
             self.assertEqual(r, 0)
 
             self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
@@ -34,8 +61,12 @@ class TestBasicFunctionality(TestCase):
         if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
             (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
                                         "deepstate.out", 1800)
+            self.assertEqual(r, 0)
 
             self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
             self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
+            self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)            
             self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
+            self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)            
             self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
+