Merge pull request #68 from trailofbits/add_more_tests

Additional tests and checks for existing tests

.travis.yml

@@ -7,9 +7,9 @@ install:
 - pip install pyflakes
 - sudo apt-get -y update
 - sudo apt-get -y install build-essential gcc-multilib cmake python python-pip python-setuptools libffi-dev
-- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip install -U pip ; sudo pip install manticore ; fi
-- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip uninstall -y Manticore || echo "Manticore not cached" ; fi
-- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip install https://github.com/trailofbits/manticore/archive/master.zip ; fi
+- sudo pip install -U pip ; sudo pip install manticore
+- sudo pip uninstall -y Manticore || echo "Manticore not cached"
+- sudo pip install https://github.com/trailofbits/manticore/archive/master.zip
 - mkdir build
 - cd build
 - cmake ..
@@ -17,13 +17,27 @@ install:
 - python setup.py install
 - cd ..
 env:
-- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr
-- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore
-- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr
-- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore
-- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr
-- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore
+- TASK=ARITHMETIC
+- TASK=CRASH
+- TASK=FIXTURE
+- TASK=KLEE
+- TASK=LISTS
+- TASK=ONEOF
+- TASK=OVERFLOW
+- TASK=PRIMES
+- TASK=STREAMINGANDFORMATTING
+- TASK=TAKEOVER
 script:
 - pyflakes bin/deepstate/*.py
 - pyflakes tests/*.py
-- nosetests --verbose
+- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic.py ; fi
+- if [ $TASK = CRASH ]; then nosetests tests/test_crash.py ; fi
+- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture.py ; fi
+- if [ $TASK = KLEE ]; then nosetests tests/test_klee.py ; fi
+- if [ $TASK = LISTS ]; then nosetests tests/test_lists.py ; fi
+- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof.py ; fi
+- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow.py ; fi
+- if [ $TASK = PRIMES ]; then nosetests tests/test_primes.py ; fi
+- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting.py ; fi
+- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover.py ; fi
+

src/include/deepstate/DeepState.hpp

@@ -163,8 +163,10 @@ class SymbolicLinearContainer {
     }
   }
 
-  DEEPSTATE_INLINE SymbolicLinearContainer(void)
-      : SymbolicLinearContainer(DeepState_SizeInRange(0, 32)) {}
+  DEEPSTATE_INLINE SymbolicLinearContainer(void) {
+    value.reserve(32);
+    value.resize(DeepState_SizeInRange(0, 32));  // Avoids symbolic `malloc`.
+  }
 
   DEEPSTATE_INLINE operator T (void) const {
     return value;

tests/deepstate_base.py

@@ -0,0 +1,13 @@
+from __future__ import print_function
+from unittest import TestCase
+
+
+class DeepStateTestCase(TestCase):
+  def test_angr(self):
+    self.run_deepstate("deepstate-angr")
+
+  def test_manticore(self):
+    self.run_deepstate("deepstate-manticore")
+
+  def run_deepstate(self, deepstate):
+    print("define an actual test of DeepState here.")

tests/logrun.py

@@ -4,20 +4,44 @@ import time
 import sys
 
 def logrun(cmd, file, timeout):
-    with open(file, 'w') as outf:
-        p = subprocess.Popen(cmd, stdout=outf, stderr=outf)
-    start = time.time()
-    oldContents = ""
-    while (p.poll() is None) and ((time.time() - start) < timeout):
-        with open(file, 'r') as inf:            
-            contents = inf.read()
-        if len(contents) > len(oldContents):
-            sys.stderr.write(contents[len(oldContents):])
-            sys.stderr.flush()
-            oldContents = contents
-        time.sleep(1)
-    sys.stderr.write("\n")
-    if p.poll() is None:
-        return ("TIMEOUT", contents)
-    return (p.returncode, contents)
-        
+  sys.stderr.write("\n\n" + ("=" * 80) + "\n")
+  sys.stderr.write("RUNNING: ")
+  sys.stderr.write(" ".join(cmd) + "\n\n")
+  sys.stderr.flush()
+  with open(file, 'w') as outf:
+    p = subprocess.Popen(cmd, stdout=outf, stderr=outf)
+  start = time.time()
+  oldContents = ""
+  lastOutput = time.time()
+  while (p.poll() is None) and ((time.time() - start) < timeout):
+    if (time.time() - lastOutput) > 300:
+      sys.stderr.write(".")
+      sys.stderr.flush()
+      lastOutput = time.time()
+    with open(file, 'r') as inf:
+      contents = inf.read()
+    if len(contents) > len(oldContents):
+      sys.stderr.write(contents[len(oldContents):])
+      sys.stderr.flush()
+      oldContents = contents
+      lastOutput = time.time()
+    time.sleep(0.05)
+  totalTime = time.time() - start
+  sys.stderr.write("\n")
+  rv = (p.returncode, contents)
+  if p.poll() is None:
+    rv = ("TIMEOUT", contents)
+  if "Traceback (most recent call last)" in contents:
+    rv = ("EXCEPTION RAISED", contents)
+  if "internal error" in contents:
+    rv = ("INTERNAL ERROR", contents)
+  sys.stderr.write("\nDONE\n\n")
+  sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n")
+  sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n")
+  sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n")
+  sys.stderr.write("=" * 80 + "\n")
+
+  return rv
+
+
+

tests/test_arithmetic.py

@@ -0,0 +1,17 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class ArithmeticTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
+    self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
+    self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)
+    self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
+    self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)
+    self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)

tests/test_basic_functionality.py

@@ -1,41 +0,0 @@
-from __future__ import print_function
-import os
-from unittest import TestCase
-import logrun
-
-
-class TestBasicFunctionality(TestCase):
-    def test_basic_functionality(self):
-        deepstate = os.getenv("DEEPSTATE_CMD")
-        if deepstate is None:
-            deepstate = "deepstate-angr" # default to angr in an environment without a defined command
-
-        if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
-            (r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
-                                        "deepstate.out", 1800)
-            
-            self.assertEqual(r, 0)
-
-            self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
-            self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
-
-            self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
-            self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
-
-        if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
-            (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
-                                        "deepstate.out", 1800)
-            
-            self.assertEqual(r, 0)
-
-            self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
-            self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
-
-        if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
-            (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
-                                        "deepstate.out", 1800)
-
-            self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
-            self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
-            self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
-            self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)

tests/test_crash.py

@@ -0,0 +1,17 @@
+from __future__ import print_function
+import deepstate_base
+import logrun
+
+
+class CrashTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Passed: Crash_SegFault" in output)
+    foundCrashSave = False
+    for line in output.split("\n"):
+      if ("Saving input to" in line) and (".crash" in line):
+        foundCrashSave = True
+    self.assertTrue(foundCrashSave)

tests/test_fixture.py

@@ -0,0 +1,16 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class FixtureTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Passed: MyTest_Something" in output)
+    self.assertFalse("Failed: MyTest_Something" in output)
+
+    self.assertTrue("Setting up!" in output)
+    self.assertTrue("Tearing down!" in output)

tests/test_klee.py

@@ -0,0 +1,14 @@
+from __future__ import print_function
+import deepstate_base
+import logrun
+
+
+class KleeTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("zero" in output)
+    self.assertTrue("positive" in output)
+    self.assertTrue("negative" in output)

tests/test_lists.py

@@ -0,0 +1,13 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class ListsTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
+                  "deepstate.out", 3000)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Passed: Vector_DoubleReversal" in output)
+    self.assertFalse("Failed: Vector_DoubleReversal" in output)

tests/test_oneof.py

@@ -0,0 +1,13 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class OneOfTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
+    self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)

tests/test_overflow.py

@@ -0,0 +1,15 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class OverflowTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output)
+    self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output)
+    self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output)
+    self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output)

tests/test_primes.py

@@ -0,0 +1,16 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class PrimesTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
+    self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
+
+    self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
+    self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)

tests/test_streamingandformatting.py

@@ -0,0 +1,25 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class StreamingAndFormattingTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"],
+                  "deepstate.out", 1800)
+    #self.assertEqual(r, 0)
+
+    self.assertTrue("Failed: Streaming_BasicLevels" in output)
+    self.assertTrue("This is a debug message" in output)
+    self.assertTrue("This is an info message" in output)
+    self.assertTrue("This is a warning message" in output)
+    self.assertTrue("This is a error message" in output)
+    self.assertTrue("This is a info message again" in output)
+    self.assertTrue(": 97" in output)
+    self.assertTrue(": 1" in output)
+    self.assertTrue(": 1.000000" in output)
+    self.assertTrue(": string" in output)
+    self.assertTrue("hello string=world" in output)
+    self.assertTrue("hello again!" in output)
+    self.assertTrue("Passed: Formatting_OverridePrintf" in output)
+    self.assertFalse("Failed: Formatting_OverridePrintf" in output)

tests/test_takeover.py

@@ -0,0 +1,20 @@
+from __future__ import print_function
+import logrun
+import deepstate_base
+
+
+class TakeOverTest(deepstate_base.DeepStateTestCase):
+  def run_deepstate(self, deepstate):
+    (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"],
+                  "deepstate.out", 1800)
+    self.assertEqual(r, 0)
+
+    self.assertTrue("hi" in output)
+    self.assertTrue("bye" in output)
+    self.assertTrue("was not greater than" in output)
+
+    foundPassSave = False
+    for line in output.split("\n"):
+      if ("Saving input to" in line) and (".pass" in line):
+        foundPassSave = True
+    self.assertTrue(foundPassSave)