Merge branch 'add_more_tests' of github.com:trailofbits/deepstate into add_more_tests
This commit is contained in:
Peter Goodman
14
.travis.yml
14
.travis.yml
@@ -17,10 +17,24 @@ install:
|
||||
- python setup.py install
|
||||
- cd ..
|
||||
env:
|
||||
- TASK=CRASH DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=KLEE DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=FIXTURE DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=LISTS DEEPSTATE_CMD=deepstate-angr
|
||||
#- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-manticore
|
||||
- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr
|
||||
- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore
|
||||
script:
|
||||
|
||||
@@ -4,20 +4,44 @@ import time
|
||||
import sys
|
||||
|
||||
def logrun(cmd, file, timeout):
|
||||
sys.stderr.write("\n\n" + ("=" * 80) + "\n")
|
||||
sys.stderr.write("RUNNING: ")
|
||||
sys.stderr.write(" ".join(cmd) + "\n\n")
|
||||
sys.stderr.flush()
|
||||
with open(file, 'w') as outf:
|
||||
p = subprocess.Popen(cmd, stdout=outf, stderr=outf)
|
||||
start = time.time()
|
||||
oldContents = ""
|
||||
lastOutput = time.time()
|
||||
while (p.poll() is None) and ((time.time() - start) < timeout):
|
||||
if (time.time() - lastOutput) > 300:
|
||||
sys.stderr.write(".")
|
||||
sys.stderr.flush()
|
||||
lastOutput = time.time()
|
||||
with open(file, 'r') as inf:
|
||||
contents = inf.read()
|
||||
if len(contents) > len(oldContents):
|
||||
sys.stderr.write(contents[len(oldContents):])
|
||||
sys.stderr.flush()
|
||||
oldContents = contents
|
||||
time.sleep(1)
|
||||
lastOutput = time.time()
|
||||
time.sleep(0.05)
|
||||
totalTime = time.time() - start
|
||||
sys.stderr.write("\n")
|
||||
rv = (p.returncode, contents)
|
||||
if p.poll() is None:
|
||||
return ("TIMEOUT", contents)
|
||||
return (p.returncode, contents)
|
||||
rv = ("TIMEOUT", contents)
|
||||
if "Traceback (most recent call last)" in contents:
|
||||
rv = ("EXCEPTION RAISED", contents)
|
||||
if "internal error" in contents:
|
||||
rv = ("INTERNAL ERROR", contents)
|
||||
sys.stderr.write("\nDONE\n\n")
|
||||
sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n")
|
||||
sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n")
|
||||
sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n")
|
||||
sys.stderr.write("=" * 80 + "\n")
|
||||
|
||||
return rv
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -6,36 +6,127 @@ import logrun
|
||||
|
||||
class TestBasicFunctionality(TestCase):
|
||||
def test_basic_functionality(self):
|
||||
deepstate = os.getenv("DEEPSTATE_CMD")
|
||||
if deepstate is None:
|
||||
deepstate = "deepstate-angr" # default to angr in an environment without a defined command
|
||||
if os.getenv("DEEPSTATE_CMD") is not None:
|
||||
deepstates = [os.getenv("DEEPSTATE_CMD")]
|
||||
else:
|
||||
deepstates = ["deepstate-angr", "deepstate-manticore"]
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
|
||||
"deepstate.out", 1800)
|
||||
|
||||
self.assertEqual(r, 0)
|
||||
print("RUNNING WITH DEEPSTATE COMMANDS:", deepstates)
|
||||
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
for deepstate in deepstates:
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
self.assertTrue("Passed: Crash_SegFault" in output)
|
||||
foundCrashSave = False
|
||||
for line in output.split("\n"):
|
||||
if ("Saving input to" in line) and (".crash" in line):
|
||||
foundCrashSave = True
|
||||
self.assertTrue(foundCrashSave)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
|
||||
"deepstate.out", 1800)
|
||||
|
||||
self.assertEqual(r, 0)
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
self.assertTrue("zero" in output)
|
||||
self.assertTrue("positive" in output)
|
||||
self.assertTrue("negative" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
|
||||
"deepstate.out", 1800)
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
|
||||
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "TAKEOVER":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("hi" in output)
|
||||
self.assertTrue("bye" in output)
|
||||
self.assertTrue("was not greater than" in output)
|
||||
|
||||
foundPassSave = False
|
||||
for line in output.split("\n"):
|
||||
if ("Saving input to" in line) and (".pass" in line):
|
||||
foundPassSave = True
|
||||
self.assertTrue(foundPassSave)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "FIXTURE":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Fixture"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Passed: MyTest_Something" in output)
|
||||
self.assertFalse("Failed: MyTest_Something" in output)
|
||||
|
||||
self.assertTrue("Setting up!" in output)
|
||||
self.assertTrue("Tearing down!" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
|
||||
"deepstate.out", 3000)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Passed: Vector_DoubleReversal" in output)
|
||||
self.assertFalse("Failed: Vector_DoubleReversal" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "STREAMINGANDFORMATTING":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"],
|
||||
"deepstate.out", 1800)
|
||||
#self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: Streaming_BasicLevels" in output)
|
||||
self.assertTrue("This is a debug message" in output)
|
||||
self.assertTrue("This is an info message" in output)
|
||||
self.assertTrue("This is a warning message" in output)
|
||||
self.assertTrue("This is a error message" in output)
|
||||
self.assertTrue("This is a info message again" in output)
|
||||
self.assertTrue(": 97" in output)
|
||||
self.assertTrue(": 1" in output)
|
||||
self.assertTrue(": 1.000000" in output)
|
||||
self.assertTrue(": string" in output)
|
||||
self.assertTrue("hello string=world" in output)
|
||||
self.assertTrue("hello again!" in output)
|
||||
self.assertTrue("Passed: Formatting_OverridePrintf" in output)
|
||||
self.assertFalse("Failed: Formatting_OverridePrintf" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "OVERFLOW":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output)
|
||||
self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output)
|
||||
self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output)
|
||||
self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output)
|
||||
|
||||
if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
|
||||
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
|
||||
"deepstate.out", 1800)
|
||||
self.assertEqual(r, 0)
|
||||
|
||||
self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
|
||||
self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
|
||||
self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
|
||||
|
||||
Reference in New Issue
Block a user