diff --git a/.travis.yml b/.travis.yml index 45b70c7..ffcfecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,9 @@ install: - pip install pyflakes - sudo apt-get -y update - sudo apt-get -y install build-essential gcc-multilib cmake python python-pip python-setuptools libffi-dev -- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip install -U pip ; sudo pip install manticore ; fi -- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip uninstall -y Manticore || echo "Manticore not cached" ; fi -- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip install https://github.com/trailofbits/manticore/archive/master.zip ; fi +- sudo pip install -U pip ; sudo pip install manticore +- sudo pip uninstall -y Manticore || echo "Manticore not cached" +- sudo pip install https://github.com/trailofbits/manticore/archive/master.zip - mkdir build - cd build - cmake .. @@ -17,13 +17,27 @@ install: - python setup.py install - cd .. env: -- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr -- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore -- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr -- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore -- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr -- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore +- TASK=ARITHMETIC +- TASK=CRASH +- TASK=FIXTURE +- TASK=KLEE +- TASK=LISTS +- TASK=ONEOF +- TASK=OVERFLOW +- TASK=PRIMES +- TASK=STREAMINGANDFORMATTING +- TASK=TAKEOVER script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py -- nosetests --verbose +- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic.py ; fi +- if [ $TASK = CRASH ]; then nosetests tests/test_crash.py ; fi +- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture.py ; fi +- if [ $TASK = KLEE ]; then nosetests tests/test_klee.py ; fi +- if [ $TASK = LISTS ]; then nosetests tests/test_lists.py ; fi +- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof.py ; fi +- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow.py ; fi +- if [ $TASK = PRIMES ]; then nosetests tests/test_primes.py ; fi +- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting.py ; fi +- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover.py ; fi + diff --git a/src/include/deepstate/DeepState.hpp b/src/include/deepstate/DeepState.hpp index 208fecc..f5572cd 100644 --- a/src/include/deepstate/DeepState.hpp +++ b/src/include/deepstate/DeepState.hpp @@ -163,8 +163,10 @@ class SymbolicLinearContainer { } } - DEEPSTATE_INLINE SymbolicLinearContainer(void) - : SymbolicLinearContainer(DeepState_SizeInRange(0, 32)) {} + DEEPSTATE_INLINE SymbolicLinearContainer(void) { + value.reserve(32); + value.resize(DeepState_SizeInRange(0, 32)); // Avoids symbolic `malloc`. + } DEEPSTATE_INLINE operator T (void) const { return value; diff --git a/tests/deepstate_base.py b/tests/deepstate_base.py new file mode 100644 index 0000000..ca2249f --- /dev/null +++ b/tests/deepstate_base.py @@ -0,0 +1,13 @@ +from __future__ import print_function +from unittest import TestCase + + +class DeepStateTestCase(TestCase): + def test_angr(self): + self.run_deepstate("deepstate-angr") + + def test_manticore(self): + self.run_deepstate("deepstate-manticore") + + def run_deepstate(self, deepstate): + print("define an actual test of DeepState here.") diff --git a/tests/logrun.py b/tests/logrun.py index d60185c..3c9c5c4 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -4,20 +4,44 @@ import time import sys def logrun(cmd, file, timeout): - with open(file, 'w') as outf: - p = subprocess.Popen(cmd, stdout=outf, stderr=outf) - start = time.time() - oldContents = "" - while (p.poll() is None) and ((time.time() - start) < timeout): - with open(file, 'r') as inf: - contents = inf.read() - if len(contents) > len(oldContents): - sys.stderr.write(contents[len(oldContents):]) - sys.stderr.flush() - oldContents = contents - time.sleep(1) - sys.stderr.write("\n") - if p.poll() is None: - return ("TIMEOUT", contents) - return (p.returncode, contents) - + sys.stderr.write("\n\n" + ("=" * 80) + "\n") + sys.stderr.write("RUNNING: ") + sys.stderr.write(" ".join(cmd) + "\n\n") + sys.stderr.flush() + with open(file, 'w') as outf: + p = subprocess.Popen(cmd, stdout=outf, stderr=outf) + start = time.time() + oldContents = "" + lastOutput = time.time() + while (p.poll() is None) and ((time.time() - start) < timeout): + if (time.time() - lastOutput) > 300: + sys.stderr.write(".") + sys.stderr.flush() + lastOutput = time.time() + with open(file, 'r') as inf: + contents = inf.read() + if len(contents) > len(oldContents): + sys.stderr.write(contents[len(oldContents):]) + sys.stderr.flush() + oldContents = contents + lastOutput = time.time() + time.sleep(0.05) + totalTime = time.time() - start + sys.stderr.write("\n") + rv = (p.returncode, contents) + if p.poll() is None: + rv = ("TIMEOUT", contents) + if "Traceback (most recent call last)" in contents: + rv = ("EXCEPTION RAISED", contents) + if "internal error" in contents: + rv = ("INTERNAL ERROR", contents) + sys.stderr.write("\nDONE\n\n") + sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") + sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") + sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n") + sys.stderr.write("=" * 80 + "\n") + + return rv + + + diff --git a/tests/test_arithmetic.py b/tests/test_arithmetic.py new file mode 100644 index 0000000..81b8593 --- /dev/null +++ b/tests/test_arithmetic.py @@ -0,0 +1,17 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class ArithmeticTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) + self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py deleted file mode 100644 index 42966fd..0000000 --- a/tests/test_basic_functionality.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import print_function -import os -from unittest import TestCase -import logrun - - -class TestBasicFunctionality(TestCase): - def test_basic_functionality(self): - deepstate = os.getenv("DEEPSTATE_CMD") - if deepstate is None: - deepstate = "deepstate-angr" # default to angr in an environment without a defined command - - if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": - (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], - "deepstate.out", 1800) - - self.assertEqual(r, 0) - - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": - (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], - "deepstate.out", 1800) - - self.assertEqual(r, 0) - - self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) - self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], - "deepstate.out", 1800) - - self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) - self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) diff --git a/tests/test_crash.py b/tests/test_crash.py new file mode 100644 index 0000000..7a503fe --- /dev/null +++ b/tests/test_crash.py @@ -0,0 +1,17 @@ +from __future__ import print_function +import deepstate_base +import logrun + + +class CrashTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) diff --git a/tests/test_fixture.py b/tests/test_fixture.py new file mode 100644 index 0000000..4d47bac --- /dev/null +++ b/tests/test_fixture.py @@ -0,0 +1,16 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class FixtureTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: MyTest_Something" in output) + self.assertFalse("Failed: MyTest_Something" in output) + + self.assertTrue("Setting up!" in output) + self.assertTrue("Tearing down!" in output) diff --git a/tests/test_klee.py b/tests/test_klee.py new file mode 100644 index 0000000..6c030af --- /dev/null +++ b/tests/test_klee.py @@ -0,0 +1,14 @@ +from __future__ import print_function +import deepstate_base +import logrun + + +class KleeTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) diff --git a/tests/test_lists.py b/tests/test_lists.py new file mode 100644 index 0000000..5c8107c --- /dev/null +++ b/tests/test_lists.py @@ -0,0 +1,13 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class ListsTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 3000) + self.assertEqual(r, 0) + + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) diff --git a/tests/test_oneof.py b/tests/test_oneof.py new file mode 100644 index 0000000..1292b7a --- /dev/null +++ b/tests/test_oneof.py @@ -0,0 +1,13 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class OneOfTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) diff --git a/tests/test_overflow.py b/tests/test_overflow.py new file mode 100644 index 0000000..f370281 --- /dev/null +++ b/tests/test_overflow.py @@ -0,0 +1,15 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class OverflowTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) + self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) diff --git a/tests/test_primes.py b/tests/test_primes.py new file mode 100644 index 0000000..05abe22 --- /dev/null +++ b/tests/test_primes.py @@ -0,0 +1,16 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class PrimesTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) diff --git a/tests/test_streamingandformatting.py b/tests/test_streamingandformatting.py new file mode 100644 index 0000000..087262f --- /dev/null +++ b/tests/test_streamingandformatting.py @@ -0,0 +1,25 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class StreamingAndFormattingTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], + "deepstate.out", 1800) + #self.assertEqual(r, 0) + + self.assertTrue("Failed: Streaming_BasicLevels" in output) + self.assertTrue("This is a debug message" in output) + self.assertTrue("This is an info message" in output) + self.assertTrue("This is a warning message" in output) + self.assertTrue("This is a error message" in output) + self.assertTrue("This is a info message again" in output) + self.assertTrue(": 97" in output) + self.assertTrue(": 1" in output) + self.assertTrue(": 1.000000" in output) + self.assertTrue(": string" in output) + self.assertTrue("hello string=world" in output) + self.assertTrue("hello again!" in output) + self.assertTrue("Passed: Formatting_OverridePrintf" in output) + self.assertFalse("Failed: Formatting_OverridePrintf" in output) diff --git a/tests/test_takeover.py b/tests/test_takeover.py new file mode 100644 index 0000000..94e39f0 --- /dev/null +++ b/tests/test_takeover.py @@ -0,0 +1,20 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class TakeOverTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("hi" in output) + self.assertTrue("bye" in output) + self.assertTrue("was not greater than" in output) + + foundPassSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".pass" in line): + foundPassSave = True + self.assertTrue(foundPassSave)