From 15ee6c849cd899cec92cc16fa00c434555cd9eff Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 18 Jul 2018 09:26:37 -0400 Subject: [PATCH] Refactor tests, and pump symbolic container sizes --- .travis.yml | 21 --- src/include/deepstate/DeepState.hpp | 68 ++++----- tests/logrun.py | 78 +++++----- tests/test_basic_functionality.py | 224 +++++++++++++++------------- 4 files changed, 193 insertions(+), 198 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7c751a..8242a7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,27 +16,6 @@ install: - make - python setup.py install - cd .. -env: -- TASK=CRASH DEEPSTATE_CMD=deepstate-angr -- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore -- TASK=KLEE DEEPSTATE_CMD=deepstate-angr -- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore -- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr -- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore -- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-angr -- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-manticore -- TASK=FIXTURE DEEPSTATE_CMD=deepstate-angr -- TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore -- TASK=LISTS DEEPSTATE_CMD=deepstate-angr -#- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore -- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-angr -- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore -- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr -- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore -- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-angr -- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-manticore -- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr -- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py diff --git a/src/include/deepstate/DeepState.hpp b/src/include/deepstate/DeepState.hpp index 208fecc..25fe0f1 100644 --- a/src/include/deepstate/DeepState.hpp +++ b/src/include/deepstate/DeepState.hpp @@ -153,40 +153,6 @@ class Symbolic { template class Symbolic {}; -template -class SymbolicLinearContainer { - public: - DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len) - : value(len) { - if (!value.empty()) { - DeepState_SymbolizeData(&(value.front()), &(value.back())); - } - } - - DEEPSTATE_INLINE SymbolicLinearContainer(void) - : SymbolicLinearContainer(DeepState_SizeInRange(0, 32)) {} - - DEEPSTATE_INLINE operator T (void) const { - return value; - } - - T value; -}; - -template <> -class Symbolic : public SymbolicLinearContainer { - using SymbolicLinearContainer::SymbolicLinearContainer; -}; - -template <> -class Symbolic : public SymbolicLinearContainer { - using SymbolicLinearContainer::SymbolicLinearContainer; -}; - -template -class Symbolic> : - public SymbolicLinearContainer> {}; - #define MAKE_SYMBOL_SPECIALIZATION(Tname, tname) \ template <> \ class Symbolic { \ @@ -313,6 +279,40 @@ static T Pump(T val, unsigned max=10) { return Minimize(val); } +template +class SymbolicLinearContainer { + public: + DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len) + : value(len) { + if (!value.empty()) { + DeepState_SymbolizeData(&(value.front()), &(value.back())); + } + } + + DEEPSTATE_INLINE SymbolicLinearContainer(void) + : SymbolicLinearContainer(Pump(DeepState_SizeInRange(0, 32), 32)) {} + + DEEPSTATE_INLINE operator T (void) const { + return value; + } + + T value; +}; + +template <> +class Symbolic : public SymbolicLinearContainer { + using SymbolicLinearContainer::SymbolicLinearContainer; +}; + +template <> +class Symbolic : public SymbolicLinearContainer { + using SymbolicLinearContainer::SymbolicLinearContainer; +}; + +template +class Symbolic> : + public SymbolicLinearContainer> {}; + template inline static void ForAll(void (*func)(Args...)) { func(Symbolic()...); diff --git a/tests/logrun.py b/tests/logrun.py index 1165b31..3c9c5c4 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -4,44 +4,44 @@ import time import sys def logrun(cmd, file, timeout): - sys.stderr.write("\n\n" + ("=" * 80) + "\n") - sys.stderr.write("RUNNING: ") - sys.stderr.write(" ".join(cmd) + "\n\n") - sys.stderr.flush() - with open(file, 'w') as outf: - p = subprocess.Popen(cmd, stdout=outf, stderr=outf) - start = time.time() - oldContents = "" - lastOutput = time.time() - while (p.poll() is None) and ((time.time() - start) < timeout): - if (time.time() - lastOutput) > 300: - sys.stderr.write(".") - sys.stderr.flush() - lastOutput = time.time() - with open(file, 'r') as inf: - contents = inf.read() - if len(contents) > len(oldContents): - sys.stderr.write(contents[len(oldContents):]) - sys.stderr.flush() - oldContents = contents - lastOutput = time.time() - time.sleep(0.05) - totalTime = time.time() - start - sys.stderr.write("\n") - rv = (p.returncode, contents) - if p.poll() is None: - rv = ("TIMEOUT", contents) - if "Traceback (most recent call last)" in contents: - rv = ("EXCEPTION RAISED", contents) - if "internal error" in contents: - rv = ("INTERNAL ERROR", contents) - sys.stderr.write("\nDONE\n\n") - sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") - sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") - sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n") - sys.stderr.write("=" * 80 + "\n") + sys.stderr.write("\n\n" + ("=" * 80) + "\n") + sys.stderr.write("RUNNING: ") + sys.stderr.write(" ".join(cmd) + "\n\n") + sys.stderr.flush() + with open(file, 'w') as outf: + p = subprocess.Popen(cmd, stdout=outf, stderr=outf) + start = time.time() + oldContents = "" + lastOutput = time.time() + while (p.poll() is None) and ((time.time() - start) < timeout): + if (time.time() - lastOutput) > 300: + sys.stderr.write(".") + sys.stderr.flush() + lastOutput = time.time() + with open(file, 'r') as inf: + contents = inf.read() + if len(contents) > len(oldContents): + sys.stderr.write(contents[len(oldContents):]) + sys.stderr.flush() + oldContents = contents + lastOutput = time.time() + time.sleep(0.05) + totalTime = time.time() - start + sys.stderr.write("\n") + rv = (p.returncode, contents) + if p.poll() is None: + rv = ("TIMEOUT", contents) + if "Traceback (most recent call last)" in contents: + rv = ("EXCEPTION RAISED", contents) + if "internal error" in contents: + rv = ("INTERNAL ERROR", contents) + sys.stderr.write("\nDONE\n\n") + sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") + sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") + sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n") + sys.stderr.write("=" * 80 + "\n") + + return rv + - return rv - - diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 5f3c270..342376e 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -4,129 +4,145 @@ from unittest import TestCase import logrun -class TestBasicFunctionality(TestCase): - def test_basic_functionality(self): - if os.getenv("DEEPSTATE_CMD") is not None: - deepstates = [os.getenv("DEEPSTATE_CMD")] - else: - deepstates = ["deepstate-angr", "deepstate-manticore"] +class DeepStateTestCase(TestCase): + def test_angr(self): + self.run_test(self, "deepstate-angr") - print("RUNNING WITH DEEPSTATE COMMANDS:", deepstates) + def test_manticore(self): + self.run_test(self, "deepstate-manticore") - for deepstate in deepstates: - if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": - (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - self.assertTrue("Passed: Crash_SegFault" in output) - foundCrashSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".crash" in line): - foundCrashSave = True - self.assertTrue(foundCrashSave) +class CrashTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": - (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) - self.assertTrue("zero" in output) - self.assertTrue("positive" in output) - self.assertTrue("negative" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": - (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], - "deepstate.out", 1800) - self.assertEqual(r, 0) +class KleeTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "TAKEOVER": - (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - self.assertTrue("hi" in output) - self.assertTrue("bye" in output) - self.assertTrue("was not greater than" in output) +class PrimesTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - foundPassSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".pass" in line): - foundPassSave = True - self.assertTrue(foundPassSave) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "FIXTURE": - (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - self.assertTrue("Passed: MyTest_Something" in output) - self.assertFalse("Failed: MyTest_Something" in output) - self.assertTrue("Setting up!" in output) - self.assertTrue("Tearing down!" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": - (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], - "deepstate.out", 3000) - self.assertEqual(r, 0) +class TakeOverTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Passed: Vector_DoubleReversal" in output) - self.assertFalse("Failed: Vector_DoubleReversal" in output) + self.assertTrue("hi" in output) + self.assertTrue("bye" in output) + self.assertTrue("was not greater than" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "STREAMINGANDFORMATTING": - (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], - "deepstate.out", 1800) - #self.assertEqual(r, 0) + foundPassSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".pass" in line): + foundPassSave = True + self.assertTrue(foundPassSave) - self.assertTrue("Failed: Streaming_BasicLevels" in output) - self.assertTrue("This is a debug message" in output) - self.assertTrue("This is an info message" in output) - self.assertTrue("This is a warning message" in output) - self.assertTrue("This is a error message" in output) - self.assertTrue("This is a info message again" in output) - self.assertTrue(": 97" in output) - self.assertTrue(": 1" in output) - self.assertTrue(": 1.000000" in output) - self.assertTrue(": string" in output) - self.assertTrue("hello string=world" in output) - self.assertTrue("hello again!" in output) - self.assertTrue("Passed: Formatting_OverridePrintf" in output) - self.assertFalse("Failed: Formatting_OverridePrintf" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": - (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], - "deepstate.out", 1800) - self.assertEqual(r, 0) +class FixtureTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) - self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: MyTest_Something" in output) + self.assertFalse("Failed: MyTest_Something" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "OVERFLOW": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Setting up!" in output) + self.assertTrue("Tearing down!" in output) - self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) - self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) - self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) - self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) - self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) +class ListsTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 3000) + self.assertEqual(r, 0) + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) + + +class StreamingAndFormattingTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], + "deepstate.out", 1800) + #self.assertEqual(r, 0) + + self.assertTrue("Failed: Streaming_BasicLevels" in output) + self.assertTrue("This is a debug message" in output) + self.assertTrue("This is an info message" in output) + self.assertTrue("This is a warning message" in output) + self.assertTrue("This is a error message" in output) + self.assertTrue("This is a info message again" in output) + self.assertTrue(": 97" in output) + self.assertTrue(": 1" in output) + self.assertTrue(": 1.000000" in output) + self.assertTrue(": string" in output) + self.assertTrue("hello string=world" in output) + self.assertTrue("hello again!" in output) + self.assertTrue("Passed: Formatting_OverridePrintf" in output) + self.assertFalse("Failed: Formatting_OverridePrintf" in output) + + +class OneOfTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + + +class OverflowTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) + self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) + + +class ArithmeticTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) + self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)