outside travis, run everything; also more verbose execution
This commit is contained in:
Alex Groce
+2
-2
@@ -21,10 +21,10 @@ env:
|
||||
- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=KLEE DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=LISTS DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=LISTS DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr
|
||||
|
||||
+18
-6
@@ -4,6 +4,10 @@ import time
|
||||
import sys
|
||||
|
||||
def logrun(cmd, file, timeout):
|
||||
sys.stderr.write("=" * 80 + "\n")
|
||||
sys.stderr.write("RUNNING: ")
|
||||
sys.stderr.write(" ".join(cmd) + "\n")
|
||||
sys.stderr.flush()
|
||||
with open(file, 'w') as outf:
|
||||
p = subprocess.Popen(cmd, stdout=outf, stderr=outf)
|
||||
start = time.time()
|
||||
@@ -15,13 +19,21 @@ def logrun(cmd, file, timeout):
|
||||
sys.stderr.write(contents[len(oldContents):])
|
||||
sys.stderr.flush()
|
||||
oldContents = contents
|
||||
time.sleep(1)
|
||||
time.sleep(0.05)
|
||||
totalTime = time.time() - start
|
||||
sys.stderr.write("\n")
|
||||
rv = (p.returncode, contents)
|
||||
if p.poll() is None:
|
||||
return ("TIMEOUT", contents)
|
||||
if "internal error" in contents:
|
||||
return ("INTERNAL ERROR", contents)
|
||||
rv = ("TIMEOUT", contents)
|
||||
if "Traceback (most recent call last)" in contents:
|
||||
return ("EXCEPTION RAISED", contents)
|
||||
return (p.returncode, contents)
|
||||
rv = ("EXCEPTION RAISED", contents)
|
||||
if "internal error" in contents:
|
||||
rv = ("INTERNAL ERROR", contents)
|
||||
sys.stderr.write("\nDONE\n\n")
|
||||
sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n")
|
||||
sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n")
|
||||
sys.stderr.write("RETURNING AS RESULT: " + str(rv[1]) + "\n")
|
||||
sys.stderr.write("=" * 80 + "\n")
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -6,67 +6,69 @@ import logrun
|
||||
|
||||
class TestBasicFunctionality(TestCase):
|
||||
def test_basic_functionality(self):
|
||||
deepstate = os.getenv("DEEPSTATE_CMD")
|
||||
if os.getenv("DEEPSTATE_CMD") is not None:
|
||||
deepstates = [os.getenv("DEEPSTATE_CMD")]
|
||||
if deepstate is None:
|
||||
deepstate = "deepstate-angr" # default to angr in an environment without a defined command
|
||||
deepstates = ["deepstate-angr", "deepstate-manticore"]
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Passed: Crash_SegFault" in output)
|
||||
foundCrashSave = False
|
||||
for line in output.split("\n"):
|
||||
if ("Saving input to" in line) and (".crash" in line):
|
||||
foundCrashSave = True
|
||||
self.assertTrue(foundCrashSave)
|
||||
for deepstate in deepstates:
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
self.assertTrue("Passed: Crash_SegFault" in output)
|
||||
foundCrashSave = False
|
||||
for line in output.split("\n"):
|
||||
if ("Saving input to" in line) and (".crash" in line):
|
||||
foundCrashSave = True
|
||||
self.assertTrue(foundCrashSave)
|
||||
|
||||
self.assertTrue("zero" in output)
|
||||
self.assertTrue("positive" in output)
|
||||
self.assertTrue("negative" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
self.assertTrue("zero" in output)
|
||||
self.assertTrue("positive" in output)
|
||||
self.assertTrue("negative" in output)
|
||||
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
|
||||
self.assertTrue("Passed: Vector_DoubleReversal" in output)
|
||||
self.assertFalse("Failed: Vector_DoubleReversal" in output)
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
self.assertTrue("Passed: Vector_DoubleReversal" in output)
|
||||
self.assertFalse("Failed: Vector_DoubleReversal" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user