diff --git a/.travis.yml b/.travis.yml index 93732b0..5dd5701 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ env: - TASK=CRASH DEEPSTATE_CMD=deepstate-manticore - TASK=KLEE DEEPSTATE_CMD=deepstate-angr - TASK=KLEE DEEPSTATE_CMD=deepstate-manticore -- TASK=LISTS DEEPSTATE_CMD=deepstate-angr -- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=PRIMES DEEPSTATE_CMD=deepstate-angr - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore +- TASK=LISTS DEEPSTATE_CMD=deepstate-angr +- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr - TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore - TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr diff --git a/tests/logrun.py b/tests/logrun.py index 12bafe5..535a0ea 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -4,6 +4,10 @@ import time import sys def logrun(cmd, file, timeout): + sys.stderr.write("=" * 80 + "\n") + sys.stderr.write("RUNNING: ") + sys.stderr.write(" ".join(cmd) + "\n") + sys.stderr.flush() with open(file, 'w') as outf: p = subprocess.Popen(cmd, stdout=outf, stderr=outf) start = time.time() @@ -15,13 +19,21 @@ def logrun(cmd, file, timeout): sys.stderr.write(contents[len(oldContents):]) sys.stderr.flush() oldContents = contents - time.sleep(1) + time.sleep(0.05) + totalTime = time.time() - start sys.stderr.write("\n") + rv = (p.returncode, contents) if p.poll() is None: - return ("TIMEOUT", contents) - if "internal error" in contents: - return ("INTERNAL ERROR", contents) + rv = ("TIMEOUT", contents) if "Traceback (most recent call last)" in contents: - return ("EXCEPTION RAISED", contents) - return (p.returncode, contents) + rv = ("EXCEPTION RAISED", contents) + if "internal error" in contents: + rv = ("INTERNAL ERROR", contents) + sys.stderr.write("\nDONE\n\n") + sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") + sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") + sys.stderr.write("RETURNING AS RESULT: " + str(rv[1]) + "\n") + sys.stderr.write("=" * 80 + "\n") + + diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 2f16943..5f8f850 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -6,67 +6,69 @@ import logrun class TestBasicFunctionality(TestCase): def test_basic_functionality(self): - deepstate = os.getenv("DEEPSTATE_CMD") + if os.getenv("DEEPSTATE_CMD") is not None: + deepstates = [os.getenv("DEEPSTATE_CMD")] if deepstate is None: - deepstate = "deepstate-angr" # default to angr in an environment without a defined command + deepstates = ["deepstate-angr", "deepstate-manticore"] - if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": - (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Passed: Crash_SegFault" in output) - foundCrashSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".crash" in line): - foundCrashSave = True - self.assertTrue(foundCrashSave) + for deepstate in deepstates: + if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": - (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) - self.assertTrue("zero" in output) - self.assertTrue("positive" in output) - self.assertTrue("negative" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": - (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": + (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": - (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - self.assertTrue("Passed: Vector_DoubleReversal" in output) - self.assertFalse("Failed: Vector_DoubleReversal" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": - (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) - self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": + (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) - self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + + if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) + self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)