From eb150fc13c4ca3bdacbac1955bac3550f2a3711f Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 20:50:00 -0700 Subject: [PATCH 01/28] Add tests for Crash, Klee, Lists examples; detect internal errors and exceptions --- .travis.yml | 6 ++++++ tests/logrun.py | 4 ++++ tests/test_basic_functionality.py | 35 +++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45b70c7..93732b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,12 @@ install: - python setup.py install - cd .. env: +- TASK=CRASH DEEPSTATE_CMD=deepstate-angr +- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore +- TASK=KLEE DEEPSTATE_CMD=deepstate-angr +- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore +- TASK=LISTS DEEPSTATE_CMD=deepstate-angr +- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=PRIMES DEEPSTATE_CMD=deepstate-angr - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr diff --git a/tests/logrun.py b/tests/logrun.py index d60185c..12bafe5 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -19,5 +19,9 @@ def logrun(cmd, file, timeout): sys.stderr.write("\n") if p.poll() is None: return ("TIMEOUT", contents) + if "internal error" in contents: + return ("INTERNAL ERROR", contents) + if "Traceback (most recent call last)" in contents: + return ("EXCEPTION RAISED", contents) return (p.returncode, contents) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 42966fd..2f16943 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -10,10 +10,30 @@ class TestBasicFunctionality(TestCase): if deepstate is None: deepstate = "deepstate-angr" # default to angr in an environment without a defined command + if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) + + if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], "deepstate.out", 1800) - self.assertEqual(r, 0) self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) @@ -22,10 +42,17 @@ class TestBasicFunctionality(TestCase): self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], "deepstate.out", 1800) - self.assertEqual(r, 0) self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) @@ -34,8 +61,12 @@ class TestBasicFunctionality(TestCase): if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], "deepstate.out", 1800) + self.assertEqual(r, 0) self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) + From ef3308b8ed98086aba8179c0783a6d8b8d47d68e Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:08:14 -0700 Subject: [PATCH 02/28] outside travis, run everything; also more verbose execution --- .travis.yml | 4 +- tests/logrun.py | 24 +++++-- tests/test_basic_functionality.py | 104 +++++++++++++++--------------- 3 files changed, 73 insertions(+), 59 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93732b0..5dd5701 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ env: - TASK=CRASH DEEPSTATE_CMD=deepstate-manticore - TASK=KLEE DEEPSTATE_CMD=deepstate-angr - TASK=KLEE DEEPSTATE_CMD=deepstate-manticore -- TASK=LISTS DEEPSTATE_CMD=deepstate-angr -- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=PRIMES DEEPSTATE_CMD=deepstate-angr - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore +- TASK=LISTS DEEPSTATE_CMD=deepstate-angr +- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr - TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore - TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr diff --git a/tests/logrun.py b/tests/logrun.py index 12bafe5..535a0ea 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -4,6 +4,10 @@ import time import sys def logrun(cmd, file, timeout): + sys.stderr.write("=" * 80 + "\n") + sys.stderr.write("RUNNING: ") + sys.stderr.write(" ".join(cmd) + "\n") + sys.stderr.flush() with open(file, 'w') as outf: p = subprocess.Popen(cmd, stdout=outf, stderr=outf) start = time.time() @@ -15,13 +19,21 @@ def logrun(cmd, file, timeout): sys.stderr.write(contents[len(oldContents):]) sys.stderr.flush() oldContents = contents - time.sleep(1) + time.sleep(0.05) + totalTime = time.time() - start sys.stderr.write("\n") + rv = (p.returncode, contents) if p.poll() is None: - return ("TIMEOUT", contents) - if "internal error" in contents: - return ("INTERNAL ERROR", contents) + rv = ("TIMEOUT", contents) if "Traceback (most recent call last)" in contents: - return ("EXCEPTION RAISED", contents) - return (p.returncode, contents) + rv = ("EXCEPTION RAISED", contents) + if "internal error" in contents: + rv = ("INTERNAL ERROR", contents) + sys.stderr.write("\nDONE\n\n") + sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") + sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") + sys.stderr.write("RETURNING AS RESULT: " + str(rv[1]) + "\n") + sys.stderr.write("=" * 80 + "\n") + + diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 2f16943..5f8f850 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -6,67 +6,69 @@ import logrun class TestBasicFunctionality(TestCase): def test_basic_functionality(self): - deepstate = os.getenv("DEEPSTATE_CMD") + if os.getenv("DEEPSTATE_CMD") is not None: + deepstates = [os.getenv("DEEPSTATE_CMD")] if deepstate is None: - deepstate = "deepstate-angr" # default to angr in an environment without a defined command + deepstates = ["deepstate-angr", "deepstate-manticore"] - if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": - (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Passed: Crash_SegFault" in output) - foundCrashSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".crash" in line): - foundCrashSave = True - self.assertTrue(foundCrashSave) + for deepstate in deepstates: + if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": - (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) - self.assertTrue("zero" in output) - self.assertTrue("positive" in output) - self.assertTrue("negative" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": - (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": + (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": - (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - self.assertTrue("Passed: Vector_DoubleReversal" in output) - self.assertFalse("Failed: Vector_DoubleReversal" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": - (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) - self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": + (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) - self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + + if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) + self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) From 7775492adc3690eb7794fc1cd2bb91432bc23a26 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:13:54 -0700 Subject: [PATCH 03/28] fix initial decison of which deepstate to use --- tests/test_basic_functionality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 5f8f850..f6bffc5 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -8,7 +8,7 @@ class TestBasicFunctionality(TestCase): def test_basic_functionality(self): if os.getenv("DEEPSTATE_CMD") is not None: deepstates = [os.getenv("DEEPSTATE_CMD")] - if deepstate is None: + else: deepstates = ["deepstate-angr", "deepstate-manticore"] for deepstate in deepstates: From 3a35a444962658c082f53a6e790b940adb0a0ccf Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:24:02 -0700 Subject: [PATCH 04/28] fix iteration over command --- .travis.yml | 2 ++ tests/logrun.py | 2 +- tests/test_basic_functionality.py | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5dd5701..2eb85b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ env: - TASK=KLEE DEEPSTATE_CMD=deepstate-manticore - TASK=PRIMES DEEPSTATE_CMD=deepstate-angr - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore +- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-angr +- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-manticore - TASK=LISTS DEEPSTATE_CMD=deepstate-angr - TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr diff --git a/tests/logrun.py b/tests/logrun.py index 535a0ea..016dd25 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -6,7 +6,7 @@ import sys def logrun(cmd, file, timeout): sys.stderr.write("=" * 80 + "\n") sys.stderr.write("RUNNING: ") - sys.stderr.write(" ".join(cmd) + "\n") + sys.stderr.write(str(cmd) + "\n") sys.stderr.flush() with open(file, 'w') as outf: p = subprocess.Popen(cmd, stdout=outf, stderr=outf) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index f6bffc5..becf1ea 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -11,6 +11,8 @@ class TestBasicFunctionality(TestCase): else: deepstates = ["deepstate-angr", "deepstate-manticore"] + print("RUNNING WITH DEEPSTATE COMMANDS:", deepstates) + for deepstate in deepstates: if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], @@ -43,7 +45,22 @@ class TestBasicFunctionality(TestCase): self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + + if os.getenv("TASK") is None or os.getenv("TASK") == "TAKEOVER": + (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + self.assertTrue("hi" in output) + self.assertTrue("bye" in output) + self.assertTrue("was not greater than" in output) + + foundPassSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".pass" in line): + foundPassSave = True + self.assertTrue(foundPassSave) + if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], "deepstate.out", 1800) From 476cdd97fdfa7637924f3bd30b70b264b7cf20a9 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:27:38 -0700 Subject: [PATCH 05/28] fix output --- tests/logrun.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/logrun.py b/tests/logrun.py index 016dd25..6305ef2 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -4,9 +4,9 @@ import time import sys def logrun(cmd, file, timeout): - sys.stderr.write("=" * 80 + "\n") + sys.stderr.write("\n\n" + ("=" * 80) + "\n") sys.stderr.write("RUNNING: ") - sys.stderr.write(str(cmd) + "\n") + sys.stderr.write(" ".join(cmd) + "\n") sys.stderr.flush() with open(file, 'w') as outf: p = subprocess.Popen(cmd, stdout=outf, stderr=outf) From f710c38f5829cd93ff79d96893a84205104cae6a Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:28:48 -0700 Subject: [PATCH 06/28] actually return something from logrun --- tests/logrun.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/logrun.py b/tests/logrun.py index 6305ef2..870e4f4 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -33,7 +33,9 @@ def logrun(cmd, file, timeout): sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") sys.stderr.write("RETURNING AS RESULT: " + str(rv[1]) + "\n") - sys.stderr.write("=" * 80 + "\n") + sys.stderr.write("=" * 80 + "\n") + + return rv From d06233b2cfbd76bd75887d02424540a980c08e64 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:29:28 -0700 Subject: [PATCH 07/28] better spacing --- tests/logrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logrun.py b/tests/logrun.py index 870e4f4..451d69b 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -6,7 +6,7 @@ import sys def logrun(cmd, file, timeout): sys.stderr.write("\n\n" + ("=" * 80) + "\n") sys.stderr.write("RUNNING: ") - sys.stderr.write(" ".join(cmd) + "\n") + sys.stderr.write(" ".join(cmd) + "\n\n") sys.stderr.flush() with open(file, 'w') as outf: p = subprocess.Popen(cmd, stdout=outf, stderr=outf) From ec52c6e27defb8ac80fc9d8c07b392a0d6a55e23 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Fri, 13 Jul 2018 21:30:19 -0700 Subject: [PATCH 08/28] print right return value --- tests/logrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logrun.py b/tests/logrun.py index 451d69b..b4bf8a9 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -32,7 +32,7 @@ def logrun(cmd, file, timeout): sys.stderr.write("\nDONE\n\n") sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") - sys.stderr.write("RETURNING AS RESULT: " + str(rv[1]) + "\n") + sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n") sys.stderr.write("=" * 80 + "\n") return rv From a07311706ec6d459743e824b538a7e383fbb7e2d Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 05:00:03 -0700 Subject: [PATCH 09/28] add fixture test --- .travis.yml | 2 ++ tests/test_basic_functionality.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2eb85b2..d1540b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,8 @@ env: - TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore - TASK=TAKEOVER DEEPSTATE_CMD=deepstate-angr - TASK=TAKEOVER DEEPSTATE_CMD=deepstate-manticore +- TASK=FIXTURE DEEPSTATE_CMD=deepstate-angr +- TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore - TASK=LISTS DEEPSTATE_CMD=deepstate-angr - TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index becf1ea..502605f 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -59,7 +59,18 @@ class TestBasicFunctionality(TestCase): for line in output.split("\n"): if ("Saving input to" in line) and (".pass" in line): foundPassSave = True - self.assertTrue(foundPassSave) + self.assertTrue(foundPassSave) + + if os.getenv("TASK") is None or os.getenv("TASK") == "FIXTURE": + (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: MyTest_Something" in output) + self.assertFalse("Failed: MyTest_Something" in output) + + self.assertTrue("Setting up!" in output) + self.assertTrue("Tearing down!" in output) if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], From 843608c35424a4c1fcd0a3ffdda4e1ba25d47664 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 05:20:11 -0700 Subject: [PATCH 10/28] streaming and formatting --- .travis.yml | 2 ++ tests/test_basic_functionality.py | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d1540b5..1ede2f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ env: - TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore - TASK=LISTS DEEPSTATE_CMD=deepstate-angr - TASK=LISTS DEEPSTATE_CMD=deepstate-manticore +- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-angr +- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr - TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore - TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 502605f..a0e4f01 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -78,7 +78,27 @@ class TestBasicFunctionality(TestCase): self.assertEqual(r, 0) self.assertTrue("Passed: Vector_DoubleReversal" in output) - self.assertFalse("Failed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) + + if os.getenv("TASK") is None or os.getenv("TASK") == "STREAMINGANDFORMATTING": + (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], + "deepstate.out", 1800) + #self.assertEqual(r, 0) + + self.assertTrue("Failed: Streaming_BasicLevels" in output) + self.assertTrue("This is a debug message" in output) + self.assertTrue("This is an info message" in output) + self.assertTrue("This is a warning message" in output) + self.assertTrue("This is a error message" in output) + self.assertTrue("This is a info message again" in output) + self.assertTrue(": 97" in output) + self.assertTrue(": 1" in output) + self.assertTrue(": 1.000000" in output) + self.assertTrue(": string" in output) + self.assertTrue(": hello string=world" in output) + self.assertTrue(": hello again!" in output) + self.assertTrue("Passed: Formatting_OverridePrintf" in output) + self.assertFalse("Failed: Formatting_OverridePrintf" in output) if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], From 8713846044ea4e3f4cca202f0f040787afeb9447 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 05:58:57 -0700 Subject: [PATCH 11/28] fix message check --- tests/test_basic_functionality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index a0e4f01..9eeea84 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -95,8 +95,8 @@ class TestBasicFunctionality(TestCase): self.assertTrue(": 1" in output) self.assertTrue(": 1.000000" in output) self.assertTrue(": string" in output) - self.assertTrue(": hello string=world" in output) - self.assertTrue(": hello again!" in output) + self.assertTrue("hello string=world" in output) + self.assertTrue("hello again!" in output) self.assertTrue("Passed: Formatting_OverridePrintf" in output) self.assertFalse("Failed: Formatting_OverridePrintf" in output) From 96fc87f78199198089e1d1e7323e5648d936057d Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 06:27:48 -0700 Subject: [PATCH 12/28] keep alive long silent tests, add integer overflow --- .travis.yml | 2 ++ tests/logrun.py | 5 +++++ tests/test_basic_functionality.py | 10 ++++++++++ 3 files changed, 17 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1ede2f5..fded504 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,8 @@ env: - TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr - TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore +- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-angr +- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-manticore - TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr - TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore script: diff --git a/tests/logrun.py b/tests/logrun.py index b4bf8a9..b5b8265 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -12,13 +12,18 @@ def logrun(cmd, file, timeout): p = subprocess.Popen(cmd, stdout=outf, stderr=outf) start = time.time() oldContents = "" + lastOutput = time.time() while (p.poll() is None) and ((time.time() - start) < timeout): + if (time.time() - lastOutput) > 300: + sys.stderr.write("\n\n** FIVE MINUTE KEEPALIVE FROM TEST LOGGING **\n\n") + sys.stderr.flush() with open(file, 'r') as inf: contents = inf.read() if len(contents) > len(oldContents): sys.stderr.write(contents[len(oldContents):]) sys.stderr.flush() oldContents = contents + lastOutput = time.time() time.sleep(0.05) totalTime = time.time() - start sys.stderr.write("\n") diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 9eeea84..f59ba96 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -108,6 +108,16 @@ class TestBasicFunctionality(TestCase): self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "OVERFLOW": + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) + self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) + if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], "deepstate.out", 1800) From 6ffd9b44861a5f42b3541c345551639e62f2d5a3 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 11:29:45 -0700 Subject: [PATCH 13/28] give Lists more time, less verbose keepalive --- tests/logrun.py | 2 +- tests/test_basic_functionality.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/logrun.py b/tests/logrun.py index b5b8265..6fc7efb 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -15,7 +15,7 @@ def logrun(cmd, file, timeout): lastOutput = time.time() while (p.poll() is None) and ((time.time() - start) < timeout): if (time.time() - lastOutput) > 300: - sys.stderr.write("\n\n** FIVE MINUTE KEEPALIVE FROM TEST LOGGING **\n\n") + sys.stderr.write(".") sys.stderr.flush() with open(file, 'r') as inf: contents = inf.read() diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index f59ba96..5f3c270 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -74,7 +74,7 @@ class TestBasicFunctionality(TestCase): if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], - "deepstate.out", 1800) + "deepstate.out", 3000) self.assertEqual(r, 0) self.assertTrue("Passed: Vector_DoubleReversal" in output) From 375bfb423542e76906833eec76a4863a9b015e43 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 12:00:47 -0700 Subject: [PATCH 14/28] fix keepalive to only output at five minute intervals --- tests/logrun.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/logrun.py b/tests/logrun.py index 6fc7efb..1165b31 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -17,6 +17,7 @@ def logrun(cmd, file, timeout): if (time.time() - lastOutput) > 300: sys.stderr.write(".") sys.stderr.flush() + lastOutput = time.time() with open(file, 'r') as inf: contents = inf.read() if len(contents) > len(oldContents): From 80453f4650f13595e770b90d5022fb5d0710758b Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Sat, 14 Jul 2018 14:23:13 -0700 Subject: [PATCH 15/28] skip manticore Lists test for now, since it times out on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fded504..b7c751a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ env: - TASK=FIXTURE DEEPSTATE_CMD=deepstate-angr - TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore - TASK=LISTS DEEPSTATE_CMD=deepstate-angr -- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore +#- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore - TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-angr - TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore - TASK=ONEOF DEEPSTATE_CMD=deepstate-angr From 15ee6c849cd899cec92cc16fa00c434555cd9eff Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 18 Jul 2018 09:26:37 -0400 Subject: [PATCH 16/28] Refactor tests, and pump symbolic container sizes --- .travis.yml | 21 --- src/include/deepstate/DeepState.hpp | 68 ++++----- tests/logrun.py | 78 +++++----- tests/test_basic_functionality.py | 224 +++++++++++++++------------- 4 files changed, 193 insertions(+), 198 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7c751a..8242a7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,27 +16,6 @@ install: - make - python setup.py install - cd .. -env: -- TASK=CRASH DEEPSTATE_CMD=deepstate-angr -- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore -- TASK=KLEE DEEPSTATE_CMD=deepstate-angr -- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore -- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr -- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore -- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-angr -- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-manticore -- TASK=FIXTURE DEEPSTATE_CMD=deepstate-angr -- TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore -- TASK=LISTS DEEPSTATE_CMD=deepstate-angr -#- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore -- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-angr -- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore -- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr -- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore -- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-angr -- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-manticore -- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr -- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py diff --git a/src/include/deepstate/DeepState.hpp b/src/include/deepstate/DeepState.hpp index 208fecc..25fe0f1 100644 --- a/src/include/deepstate/DeepState.hpp +++ b/src/include/deepstate/DeepState.hpp @@ -153,40 +153,6 @@ class Symbolic { template class Symbolic {}; -template -class SymbolicLinearContainer { - public: - DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len) - : value(len) { - if (!value.empty()) { - DeepState_SymbolizeData(&(value.front()), &(value.back())); - } - } - - DEEPSTATE_INLINE SymbolicLinearContainer(void) - : SymbolicLinearContainer(DeepState_SizeInRange(0, 32)) {} - - DEEPSTATE_INLINE operator T (void) const { - return value; - } - - T value; -}; - -template <> -class Symbolic : public SymbolicLinearContainer { - using SymbolicLinearContainer::SymbolicLinearContainer; -}; - -template <> -class Symbolic : public SymbolicLinearContainer { - using SymbolicLinearContainer::SymbolicLinearContainer; -}; - -template -class Symbolic> : - public SymbolicLinearContainer> {}; - #define MAKE_SYMBOL_SPECIALIZATION(Tname, tname) \ template <> \ class Symbolic { \ @@ -313,6 +279,40 @@ static T Pump(T val, unsigned max=10) { return Minimize(val); } +template +class SymbolicLinearContainer { + public: + DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len) + : value(len) { + if (!value.empty()) { + DeepState_SymbolizeData(&(value.front()), &(value.back())); + } + } + + DEEPSTATE_INLINE SymbolicLinearContainer(void) + : SymbolicLinearContainer(Pump(DeepState_SizeInRange(0, 32), 32)) {} + + DEEPSTATE_INLINE operator T (void) const { + return value; + } + + T value; +}; + +template <> +class Symbolic : public SymbolicLinearContainer { + using SymbolicLinearContainer::SymbolicLinearContainer; +}; + +template <> +class Symbolic : public SymbolicLinearContainer { + using SymbolicLinearContainer::SymbolicLinearContainer; +}; + +template +class Symbolic> : + public SymbolicLinearContainer> {}; + template inline static void ForAll(void (*func)(Args...)) { func(Symbolic()...); diff --git a/tests/logrun.py b/tests/logrun.py index 1165b31..3c9c5c4 100644 --- a/tests/logrun.py +++ b/tests/logrun.py @@ -4,44 +4,44 @@ import time import sys def logrun(cmd, file, timeout): - sys.stderr.write("\n\n" + ("=" * 80) + "\n") - sys.stderr.write("RUNNING: ") - sys.stderr.write(" ".join(cmd) + "\n\n") - sys.stderr.flush() - with open(file, 'w') as outf: - p = subprocess.Popen(cmd, stdout=outf, stderr=outf) - start = time.time() - oldContents = "" - lastOutput = time.time() - while (p.poll() is None) and ((time.time() - start) < timeout): - if (time.time() - lastOutput) > 300: - sys.stderr.write(".") - sys.stderr.flush() - lastOutput = time.time() - with open(file, 'r') as inf: - contents = inf.read() - if len(contents) > len(oldContents): - sys.stderr.write(contents[len(oldContents):]) - sys.stderr.flush() - oldContents = contents - lastOutput = time.time() - time.sleep(0.05) - totalTime = time.time() - start - sys.stderr.write("\n") - rv = (p.returncode, contents) - if p.poll() is None: - rv = ("TIMEOUT", contents) - if "Traceback (most recent call last)" in contents: - rv = ("EXCEPTION RAISED", contents) - if "internal error" in contents: - rv = ("INTERNAL ERROR", contents) - sys.stderr.write("\nDONE\n\n") - sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") - sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") - sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n") - sys.stderr.write("=" * 80 + "\n") + sys.stderr.write("\n\n" + ("=" * 80) + "\n") + sys.stderr.write("RUNNING: ") + sys.stderr.write(" ".join(cmd) + "\n\n") + sys.stderr.flush() + with open(file, 'w') as outf: + p = subprocess.Popen(cmd, stdout=outf, stderr=outf) + start = time.time() + oldContents = "" + lastOutput = time.time() + while (p.poll() is None) and ((time.time() - start) < timeout): + if (time.time() - lastOutput) > 300: + sys.stderr.write(".") + sys.stderr.flush() + lastOutput = time.time() + with open(file, 'r') as inf: + contents = inf.read() + if len(contents) > len(oldContents): + sys.stderr.write(contents[len(oldContents):]) + sys.stderr.flush() + oldContents = contents + lastOutput = time.time() + time.sleep(0.05) + totalTime = time.time() - start + sys.stderr.write("\n") + rv = (p.returncode, contents) + if p.poll() is None: + rv = ("TIMEOUT", contents) + if "Traceback (most recent call last)" in contents: + rv = ("EXCEPTION RAISED", contents) + if "internal error" in contents: + rv = ("INTERNAL ERROR", contents) + sys.stderr.write("\nDONE\n\n") + sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n") + sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n") + sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n") + sys.stderr.write("=" * 80 + "\n") + + return rv + - return rv - - diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 5f3c270..342376e 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -4,129 +4,145 @@ from unittest import TestCase import logrun -class TestBasicFunctionality(TestCase): - def test_basic_functionality(self): - if os.getenv("DEEPSTATE_CMD") is not None: - deepstates = [os.getenv("DEEPSTATE_CMD")] - else: - deepstates = ["deepstate-angr", "deepstate-manticore"] +class DeepStateTestCase(TestCase): + def test_angr(self): + self.run_test(self, "deepstate-angr") - print("RUNNING WITH DEEPSTATE COMMANDS:", deepstates) + def test_manticore(self): + self.run_test(self, "deepstate-manticore") - for deepstate in deepstates: - if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH": - (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - self.assertTrue("Passed: Crash_SegFault" in output) - foundCrashSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".crash" in line): - foundCrashSave = True - self.assertTrue(foundCrashSave) +class CrashTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE": - (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) - self.assertTrue("zero" in output) - self.assertTrue("positive" in output) - self.assertTrue("negative" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES": - (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], - "deepstate.out", 1800) - self.assertEqual(r, 0) +class KleeTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "TAKEOVER": - (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - self.assertTrue("hi" in output) - self.assertTrue("bye" in output) - self.assertTrue("was not greater than" in output) +class PrimesTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - foundPassSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".pass" in line): - foundPassSave = True - self.assertTrue(foundPassSave) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "FIXTURE": - (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - self.assertTrue("Passed: MyTest_Something" in output) - self.assertFalse("Failed: MyTest_Something" in output) - self.assertTrue("Setting up!" in output) - self.assertTrue("Tearing down!" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS": - (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], - "deepstate.out", 3000) - self.assertEqual(r, 0) +class TakeOverTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Passed: Vector_DoubleReversal" in output) - self.assertFalse("Failed: Vector_DoubleReversal" in output) + self.assertTrue("hi" in output) + self.assertTrue("bye" in output) + self.assertTrue("was not greater than" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "STREAMINGANDFORMATTING": - (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], - "deepstate.out", 1800) - #self.assertEqual(r, 0) + foundPassSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".pass" in line): + foundPassSave = True + self.assertTrue(foundPassSave) - self.assertTrue("Failed: Streaming_BasicLevels" in output) - self.assertTrue("This is a debug message" in output) - self.assertTrue("This is an info message" in output) - self.assertTrue("This is a warning message" in output) - self.assertTrue("This is a error message" in output) - self.assertTrue("This is a info message again" in output) - self.assertTrue(": 97" in output) - self.assertTrue(": 1" in output) - self.assertTrue(": 1.000000" in output) - self.assertTrue(": string" in output) - self.assertTrue("hello string=world" in output) - self.assertTrue("hello again!" in output) - self.assertTrue("Passed: Formatting_OverridePrintf" in output) - self.assertFalse("Failed: Formatting_OverridePrintf" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF": - (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], - "deepstate.out", 1800) - self.assertEqual(r, 0) +class FixtureTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], + "deepstate.out", 1800) + self.assertEqual(r, 0) - self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) - self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: MyTest_Something" in output) + self.assertFalse("Failed: MyTest_Something" in output) - if os.getenv("TASK") is None or os.getenv("TASK") == "OVERFLOW": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], - "deepstate.out", 1800) - self.assertEqual(r, 0) + self.assertTrue("Setting up!" in output) + self.assertTrue("Tearing down!" in output) - self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) - self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) - self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) - self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) - - if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC": - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) - self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) +class ListsTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 3000) + self.assertEqual(r, 0) + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) + + +class StreamingAndFormattingTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], + "deepstate.out", 1800) + #self.assertEqual(r, 0) + + self.assertTrue("Failed: Streaming_BasicLevels" in output) + self.assertTrue("This is a debug message" in output) + self.assertTrue("This is an info message" in output) + self.assertTrue("This is a warning message" in output) + self.assertTrue("This is a error message" in output) + self.assertTrue("This is a info message again" in output) + self.assertTrue(": 97" in output) + self.assertTrue(": 1" in output) + self.assertTrue(": 1.000000" in output) + self.assertTrue(": string" in output) + self.assertTrue("hello string=world" in output) + self.assertTrue("hello again!" in output) + self.assertTrue("Passed: Formatting_OverridePrintf" in output) + self.assertFalse("Failed: Formatting_OverridePrintf" in output) + + +class OneOfTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) + + +class OverflowTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) + self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) + + +class ArithmeticTest(DeepStateTestCase): + def run_test(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) + self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) From c9e413cafa2ffa2fa04111020f3df9946f93654f Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 18 Jul 2018 09:35:02 -0400 Subject: [PATCH 17/28] Minor tweak to symbolically-sized containers. --- src/include/deepstate/DeepState.hpp | 70 +++++++++++++++-------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/include/deepstate/DeepState.hpp b/src/include/deepstate/DeepState.hpp index 25fe0f1..f5572cd 100644 --- a/src/include/deepstate/DeepState.hpp +++ b/src/include/deepstate/DeepState.hpp @@ -153,6 +153,42 @@ class Symbolic { template class Symbolic {}; +template +class SymbolicLinearContainer { + public: + DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len) + : value(len) { + if (!value.empty()) { + DeepState_SymbolizeData(&(value.front()), &(value.back())); + } + } + + DEEPSTATE_INLINE SymbolicLinearContainer(void) { + value.reserve(32); + value.resize(DeepState_SizeInRange(0, 32)); // Avoids symbolic `malloc`. + } + + DEEPSTATE_INLINE operator T (void) const { + return value; + } + + T value; +}; + +template <> +class Symbolic : public SymbolicLinearContainer { + using SymbolicLinearContainer::SymbolicLinearContainer; +}; + +template <> +class Symbolic : public SymbolicLinearContainer { + using SymbolicLinearContainer::SymbolicLinearContainer; +}; + +template +class Symbolic> : + public SymbolicLinearContainer> {}; + #define MAKE_SYMBOL_SPECIALIZATION(Tname, tname) \ template <> \ class Symbolic { \ @@ -279,40 +315,6 @@ static T Pump(T val, unsigned max=10) { return Minimize(val); } -template -class SymbolicLinearContainer { - public: - DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len) - : value(len) { - if (!value.empty()) { - DeepState_SymbolizeData(&(value.front()), &(value.back())); - } - } - - DEEPSTATE_INLINE SymbolicLinearContainer(void) - : SymbolicLinearContainer(Pump(DeepState_SizeInRange(0, 32), 32)) {} - - DEEPSTATE_INLINE operator T (void) const { - return value; - } - - T value; -}; - -template <> -class Symbolic : public SymbolicLinearContainer { - using SymbolicLinearContainer::SymbolicLinearContainer; -}; - -template <> -class Symbolic : public SymbolicLinearContainer { - using SymbolicLinearContainer::SymbolicLinearContainer; -}; - -template -class Symbolic> : - public SymbolicLinearContainer> {}; - template inline static void ForAll(void (*func)(Args...)) { func(Symbolic()...); From 13403cb18faa8960fdbe08ab4e7a55801dcdca3d Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 18 Jul 2018 09:38:29 -0400 Subject: [PATCH 18/28] Dumb mistake --- tests/test_basic_functionality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 342376e..6031c76 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -6,10 +6,10 @@ import logrun class DeepStateTestCase(TestCase): def test_angr(self): - self.run_test(self, "deepstate-angr") + self.run_test("deepstate-angr") def test_manticore(self): - self.run_test(self, "deepstate-manticore") + self.run_test("deepstate-manticore") class CrashTest(DeepStateTestCase): From 97f06a81bed829b78e7b32f20cb0792099c1362a Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 11:35:38 -0700 Subject: [PATCH 19/28] fix name of testing function, remove import of os --- tests/test_basic_functionality.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py index 6031c76..7a8346d 100644 --- a/tests/test_basic_functionality.py +++ b/tests/test_basic_functionality.py @@ -1,19 +1,18 @@ from __future__ import print_function -import os from unittest import TestCase import logrun class DeepStateTestCase(TestCase): def test_angr(self): - self.run_test("deepstate-angr") + self.run_deepstate("deepstate-angr") def test_manticore(self): - self.run_test("deepstate-manticore") + self.run_deepstate("deepstate-manticore") class CrashTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -27,7 +26,7 @@ class CrashTest(DeepStateTestCase): class KleeTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -38,7 +37,7 @@ class KleeTest(DeepStateTestCase): class PrimesTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -51,7 +50,7 @@ class PrimesTest(DeepStateTestCase): class TakeOverTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -68,7 +67,7 @@ class TakeOverTest(DeepStateTestCase): class FixtureTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -81,7 +80,7 @@ class FixtureTest(DeepStateTestCase): class ListsTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], "deepstate.out", 3000) self.assertEqual(r, 0) @@ -91,7 +90,7 @@ class ListsTest(DeepStateTestCase): class StreamingAndFormattingTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], "deepstate.out", 1800) #self.assertEqual(r, 0) @@ -113,7 +112,7 @@ class StreamingAndFormattingTest(DeepStateTestCase): class OneOfTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -123,7 +122,7 @@ class OneOfTest(DeepStateTestCase): class OverflowTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], "deepstate.out", 1800) self.assertEqual(r, 0) @@ -135,7 +134,7 @@ class OverflowTest(DeepStateTestCase): class ArithmeticTest(DeepStateTestCase): - def run_test(self, deepstate): + def run_deepstate(self, deepstate): (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], "deepstate.out", 1800) self.assertEqual(r, 0) From 7620fb3e42e26f41a2caf342026dd9eea1cb1d1c Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:07:23 -0700 Subject: [PATCH 20/28] refactor tests into multiple files for control in travis, easier editing --- tests/test_arithmetic.py | 17 ++++ tests/test_basic_functionality.py | 147 --------------------------- tests/test_crash.py | 17 ++++ tests/test_fixture.py | 16 +++ tests/test_klee.py | 14 +++ tests/test_lists.py | 13 +++ tests/test_oneof.py | 13 +++ tests/test_overflow.py | 15 +++ tests/test_primes.py | 16 +++ tests/test_streamingandformatting.py | 25 +++++ tests/test_takeover.py | 20 ++++ 11 files changed, 166 insertions(+), 147 deletions(-) create mode 100644 tests/test_arithmetic.py delete mode 100644 tests/test_basic_functionality.py create mode 100644 tests/test_crash.py create mode 100644 tests/test_fixture.py create mode 100644 tests/test_klee.py create mode 100644 tests/test_lists.py create mode 100644 tests/test_oneof.py create mode 100644 tests/test_overflow.py create mode 100644 tests/test_primes.py create mode 100644 tests/test_streamingandformatting.py create mode 100644 tests/test_takeover.py diff --git a/tests/test_arithmetic.py b/tests/test_arithmetic.py new file mode 100644 index 0000000..81b8593 --- /dev/null +++ b/tests/test_arithmetic.py @@ -0,0 +1,17 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class ArithmeticTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) + self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) + self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) + self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) diff --git a/tests/test_basic_functionality.py b/tests/test_basic_functionality.py deleted file mode 100644 index 7a8346d..0000000 --- a/tests/test_basic_functionality.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import print_function -from unittest import TestCase -import logrun - - -class DeepStateTestCase(TestCase): - def test_angr(self): - self.run_deepstate("deepstate-angr") - - def test_manticore(self): - self.run_deepstate("deepstate-manticore") - - -class CrashTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Passed: Crash_SegFault" in output) - foundCrashSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".crash" in line): - foundCrashSave = True - self.assertTrue(foundCrashSave) - - -class KleeTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("zero" in output) - self.assertTrue("positive" in output) - self.assertTrue("negative" in output) - - -class PrimesTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) - self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) - - -class TakeOverTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("hi" in output) - self.assertTrue("bye" in output) - self.assertTrue("was not greater than" in output) - - foundPassSave = False - for line in output.split("\n"): - if ("Saving input to" in line) and (".pass" in line): - foundPassSave = True - self.assertTrue(foundPassSave) - - -class FixtureTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Passed: MyTest_Something" in output) - self.assertFalse("Failed: MyTest_Something" in output) - - self.assertTrue("Setting up!" in output) - self.assertTrue("Tearing down!" in output) - - -class ListsTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], - "deepstate.out", 3000) - self.assertEqual(r, 0) - - self.assertTrue("Passed: Vector_DoubleReversal" in output) - self.assertFalse("Failed: Vector_DoubleReversal" in output) - - -class StreamingAndFormattingTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], - "deepstate.out", 1800) - #self.assertEqual(r, 0) - - self.assertTrue("Failed: Streaming_BasicLevels" in output) - self.assertTrue("This is a debug message" in output) - self.assertTrue("This is an info message" in output) - self.assertTrue("This is a warning message" in output) - self.assertTrue("This is a error message" in output) - self.assertTrue("This is a info message again" in output) - self.assertTrue(": 97" in output) - self.assertTrue(": 1" in output) - self.assertTrue(": 1.000000" in output) - self.assertTrue(": string" in output) - self.assertTrue("hello string=world" in output) - self.assertTrue("hello again!" in output) - self.assertTrue("Passed: Formatting_OverridePrintf" in output) - self.assertFalse("Failed: Formatting_OverridePrintf" in output) - - -class OneOfTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) - self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) - - -class OverflowTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) - self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) - self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) - self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) - - -class ArithmeticTest(DeepStateTestCase): - def run_deepstate(self, deepstate): - (r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"], - "deepstate.out", 1800) - self.assertEqual(r, 0) - - self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output) - self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output) - self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output) - self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output) diff --git a/tests/test_crash.py b/tests/test_crash.py new file mode 100644 index 0000000..7a503fe --- /dev/null +++ b/tests/test_crash.py @@ -0,0 +1,17 @@ +from __future__ import print_function +import deepstate_base +import logrun + + +class CrashTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Crash"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: Crash_SegFault" in output) + foundCrashSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".crash" in line): + foundCrashSave = True + self.assertTrue(foundCrashSave) diff --git a/tests/test_fixture.py b/tests/test_fixture.py new file mode 100644 index 0000000..4d47bac --- /dev/null +++ b/tests/test_fixture.py @@ -0,0 +1,16 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class FixtureTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Fixture"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Passed: MyTest_Something" in output) + self.assertFalse("Failed: MyTest_Something" in output) + + self.assertTrue("Setting up!" in output) + self.assertTrue("Tearing down!" in output) diff --git a/tests/test_klee.py b/tests/test_klee.py new file mode 100644 index 0000000..6c030af --- /dev/null +++ b/tests/test_klee.py @@ -0,0 +1,14 @@ +from __future__ import print_function +import deepstate_base +import logrun + + +class KleeTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("zero" in output) + self.assertTrue("positive" in output) + self.assertTrue("negative" in output) diff --git a/tests/test_lists.py b/tests/test_lists.py new file mode 100644 index 0000000..5c8107c --- /dev/null +++ b/tests/test_lists.py @@ -0,0 +1,13 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class ListsTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Lists"], + "deepstate.out", 3000) + self.assertEqual(r, 0) + + self.assertTrue("Passed: Vector_DoubleReversal" in output) + self.assertFalse("Failed: Vector_DoubleReversal" in output) diff --git a/tests/test_oneof.py b/tests/test_oneof.py new file mode 100644 index 0000000..1292b7a --- /dev/null +++ b/tests/test_oneof.py @@ -0,0 +1,13 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class OneOfTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/OneOf"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output) + self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output) diff --git a/tests/test_overflow.py b/tests/test_overflow.py new file mode 100644 index 0000000..f370281 --- /dev/null +++ b/tests/test_overflow.py @@ -0,0 +1,15 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class OverflowTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output) + self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output) + self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output) diff --git a/tests/test_primes.py b/tests/test_primes.py new file mode 100644 index 0000000..05abe22 --- /dev/null +++ b/tests/test_primes.py @@ -0,0 +1,16 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class PrimesTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/Primes"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) + + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output) + self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output) diff --git a/tests/test_streamingandformatting.py b/tests/test_streamingandformatting.py new file mode 100644 index 0000000..087262f --- /dev/null +++ b/tests/test_streamingandformatting.py @@ -0,0 +1,25 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class StreamingAndFormattingTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"], + "deepstate.out", 1800) + #self.assertEqual(r, 0) + + self.assertTrue("Failed: Streaming_BasicLevels" in output) + self.assertTrue("This is a debug message" in output) + self.assertTrue("This is an info message" in output) + self.assertTrue("This is a warning message" in output) + self.assertTrue("This is a error message" in output) + self.assertTrue("This is a info message again" in output) + self.assertTrue(": 97" in output) + self.assertTrue(": 1" in output) + self.assertTrue(": 1.000000" in output) + self.assertTrue(": string" in output) + self.assertTrue("hello string=world" in output) + self.assertTrue("hello again!" in output) + self.assertTrue("Passed: Formatting_OverridePrintf" in output) + self.assertFalse("Failed: Formatting_OverridePrintf" in output) diff --git a/tests/test_takeover.py b/tests/test_takeover.py new file mode 100644 index 0000000..94e39f0 --- /dev/null +++ b/tests/test_takeover.py @@ -0,0 +1,20 @@ +from __future__ import print_function +import logrun +import deepstate_base + + +class TakeOverTest(deepstate_base.DeepStateTestCase): + def run_deepstate(self, deepstate): + (r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"], + "deepstate.out", 1800) + self.assertEqual(r, 0) + + self.assertTrue("hi" in output) + self.assertTrue("bye" in output) + self.assertTrue("was not greater than" in output) + + foundPassSave = False + for line in output.split("\n"): + if ("Saving input to" in line) and (".pass" in line): + foundPassSave = True + self.assertTrue(foundPassSave) From 4eaa8e88c1a7fe70110b067bae80aeec7d886668 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:13:45 -0700 Subject: [PATCH 21/28] split testing into multiple tasks in travis --- .travis.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8242a7d..61d588c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,28 @@ install: - make - python setup.py install - cd .. +env: +TASK=ARITHMETIC +TASK=CRASH +TASK=FIXTURE +TASK=KLEE +TASK=LISTS +TASK=ONEOF +TASK=OVERFLOW +TASK=PRIMES +TASK=STREAMINGANDFORMATTING +TASK=TAKEOVER script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py -- nosetests --verbose +- if [ $TASK = ARITHMETIC ]; then nosetests test_arithmetic ; fi +- if [ $TASK = CRASH ]; then nosetests test_crash ; fi +- if [ $TASK = FIXTURE ]; then nosetests test_fixture ; fi +- if [ $TASK = KLEE ]; then nosetests test_klee ; fi +- if [ $TASK = LISTS ]; then nosetests test_lists ; fi +- if [ $TASK = ONEOF ]; then nosetests test_oneof ; fi +- if [ $TASK = OVERFLOW ]; then nosetests test_overflow ; fi +- if [ $TASK = PRIMES ]; then nosetests test_primes ; fi +- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests test_streamingandformatting ; fi +- if [ $TASK = TAKEOVER ]; then nosetests test_takeover ; fi + From f7013ebad9f063548da0a5e84b67343b96541f0c Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:14:09 -0700 Subject: [PATCH 22/28] add base for all tests --- tests/deepstate_base.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/deepstate_base.py diff --git a/tests/deepstate_base.py b/tests/deepstate_base.py new file mode 100644 index 0000000..d04d6e3 --- /dev/null +++ b/tests/deepstate_base.py @@ -0,0 +1,14 @@ +from __future__ import print_function +from unittest import TestCase +import logrun + + +class DeepStateTestCase(TestCase): + def test_angr(self): + self.run_deepstate("deepstate-angr") + + def test_manticore(self): + self.run_deepstate("deepstate-manticore") + + def run_deepstate(self, deepstate): + print("You should define an actual test of DeepState here.") From 7c0289ffe45defa2faaaa7f7cffa5f7de5ac232e Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:18:31 -0700 Subject: [PATCH 23/28] change message in base class --- tests/deepstate_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/deepstate_base.py b/tests/deepstate_base.py index d04d6e3..3c2a524 100644 --- a/tests/deepstate_base.py +++ b/tests/deepstate_base.py @@ -11,4 +11,4 @@ class DeepStateTestCase(TestCase): self.run_deepstate("deepstate-manticore") def run_deepstate(self, deepstate): - print("You should define an actual test of DeepState here.") + print("define an actual test of DeepState here.") From 92ed4f7c15d641240dd640ebddcdf311891975a0 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:32:25 -0700 Subject: [PATCH 24/28] fix stupid .yml file missing dash --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61d588c..4d0cefc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,16 +17,16 @@ install: - python setup.py install - cd .. env: -TASK=ARITHMETIC -TASK=CRASH -TASK=FIXTURE -TASK=KLEE -TASK=LISTS -TASK=ONEOF -TASK=OVERFLOW -TASK=PRIMES -TASK=STREAMINGANDFORMATTING -TASK=TAKEOVER +- TASK=ARITHMETIC +- TASK=CRASH +- TASK=FIXTURE +- TASK=KLEE +- TASK=LISTS +- TASK=ONEOF +- TASK=OVERFLOW +- TASK=PRIMES +- TASK=STREAMINGANDFORMATTING +- TASK=TAKEOVER script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py From ae35d9ee83efd6b62e931c665d08c819ee23284d Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:35:04 -0700 Subject: [PATCH 25/28] always need to install manticore now --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d0cefc..0958e91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,9 @@ install: - pip install pyflakes - sudo apt-get -y update - sudo apt-get -y install build-essential gcc-multilib cmake python python-pip python-setuptools libffi-dev -- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip install -U pip ; sudo pip install manticore ; fi -- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip uninstall -y Manticore || echo "Manticore not cached" ; fi -- if [ $DEEPSTATE_CMD = deepstate-manticore ]; then sudo pip install https://github.com/trailofbits/manticore/archive/master.zip ; fi +- sudo pip install -U pip ; sudo pip install manticore +- sudo pip uninstall -y Manticore || echo "Manticore not cached" +- sudo pip install https://github.com/trailofbits/manticore/archive/master.zip - mkdir build - cd build - cmake .. From 0a191916402cfe67139e87ef959209ad98dda751 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:41:00 -0700 Subject: [PATCH 26/28] add directory for nosetests command --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0958e91..e921010 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,14 @@ env: script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py -- if [ $TASK = ARITHMETIC ]; then nosetests test_arithmetic ; fi -- if [ $TASK = CRASH ]; then nosetests test_crash ; fi -- if [ $TASK = FIXTURE ]; then nosetests test_fixture ; fi -- if [ $TASK = KLEE ]; then nosetests test_klee ; fi -- if [ $TASK = LISTS ]; then nosetests test_lists ; fi -- if [ $TASK = ONEOF ]; then nosetests test_oneof ; fi -- if [ $TASK = OVERFLOW ]; then nosetests test_overflow ; fi -- if [ $TASK = PRIMES ]; then nosetests test_primes ; fi -- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests test_streamingandformatting ; fi -- if [ $TASK = TAKEOVER ]; then nosetests test_takeover ; fi +- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic ; fi +- if [ $TASK = CRASH ]; then nosetests tests/test_crash ; fi +- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture ; fi +- if [ $TASK = KLEE ]; then nosetests tests/test_klee ; fi +- if [ $TASK = LISTS ]; then nosetests tests/test_lists ; fi +- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof ; fi +- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow ; fi +- if [ $TASK = PRIMES ]; then nosetests tests/test_primes ; fi +- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting ; fi +- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover ; fi From 1cfb65fb52775a8ac6fbbe71fb3f9e0e5cf40916 Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:45:26 -0700 Subject: [PATCH 27/28] remove spurious import --- tests/deepstate_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/deepstate_base.py b/tests/deepstate_base.py index 3c2a524..ca2249f 100644 --- a/tests/deepstate_base.py +++ b/tests/deepstate_base.py @@ -1,6 +1,5 @@ from __future__ import print_function from unittest import TestCase -import logrun class DeepStateTestCase(TestCase): From 70d812eda2830365bab58eb741d948b0de499b0f Mon Sep 17 00:00:00 2001 From: Alex Groce Date: Wed, 18 Jul 2018 12:54:10 -0700 Subject: [PATCH 28/28] can't find tests --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index e921010..ffcfecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,14 @@ env: script: - pyflakes bin/deepstate/*.py - pyflakes tests/*.py -- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic ; fi -- if [ $TASK = CRASH ]; then nosetests tests/test_crash ; fi -- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture ; fi -- if [ $TASK = KLEE ]; then nosetests tests/test_klee ; fi -- if [ $TASK = LISTS ]; then nosetests tests/test_lists ; fi -- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof ; fi -- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow ; fi -- if [ $TASK = PRIMES ]; then nosetests tests/test_primes ; fi -- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting ; fi -- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover ; fi +- if [ $TASK = ARITHMETIC ]; then nosetests tests/test_arithmetic.py ; fi +- if [ $TASK = CRASH ]; then nosetests tests/test_crash.py ; fi +- if [ $TASK = FIXTURE ]; then nosetests tests/test_fixture.py ; fi +- if [ $TASK = KLEE ]; then nosetests tests/test_klee.py ; fi +- if [ $TASK = LISTS ]; then nosetests tests/test_lists.py ; fi +- if [ $TASK = ONEOF ]; then nosetests tests/test_oneof.py ; fi +- if [ $TASK = OVERFLOW ]; then nosetests tests/test_overflow.py ; fi +- if [ $TASK = PRIMES ]; then nosetests tests/test_primes.py ; fi +- if [ $TASK = STREAMINGANDFORMATTING ]; then nosetests tests/test_streamingandformatting.py ; fi +- if [ $TASK = TAKEOVER ]; then nosetests tests/test_takeover.py ; fi