DeforaNetworks/deepstate
3
0
Fork 0
Code Releases Activity

Refactor tests, and pump symbolic container sizes

Browse Source
This commit is contained in:
Peter Goodman
2018-07-18 09:26:37 -04:00
parent 4de753a619
commit 15ee6c849c
4 changed files with 193 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
.travis.yml
View File

@@ -16,27 +16,6 @@ install:
- make
- python setup.py install
- cd ..
env:
- TASK=CRASH DEEPSTATE_CMD=deepstate-angr
- TASK=CRASH DEEPSTATE_CMD=deepstate-manticore
- TASK=KLEE DEEPSTATE_CMD=deepstate-angr
- TASK=KLEE DEEPSTATE_CMD=deepstate-manticore
- TASK=PRIMES DEEPSTATE_CMD=deepstate-angr
- TASK=PRIMES DEEPSTATE_CMD=deepstate-manticore
- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-angr
- TASK=TAKEOVER DEEPSTATE_CMD=deepstate-manticore
- TASK=FIXTURE DEEPSTATE_CMD=deepstate-angr
- TASK=FIXTURE DEEPSTATE_CMD=deepstate-manticore
- TASK=LISTS DEEPSTATE_CMD=deepstate-angr
#- TASK=LISTS DEEPSTATE_CMD=deepstate-manticore
- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-angr
- TASK=STREAMINGANDFORMATTING DEEPSTATE_CMD=deepstate-manticore
- TASK=ONEOF DEEPSTATE_CMD=deepstate-angr
- TASK=ONEOF DEEPSTATE_CMD=deepstate-manticore
- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-angr
- TASK=OVERFLOW DEEPSTATE_CMD=deepstate-manticore
- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-angr
- TASK=ARITHMETIC DEEPSTATE_CMD=deepstate-manticore
script:
- pyflakes bin/deepstate/*.py
- pyflakes tests/*.py

68
src/include/deepstate/DeepState.hpp
View File

@@ -153,40 +153,6 @@ class Symbolic {
template <typename T>
class Symbolic<T &> {};
template <typename T>
class SymbolicLinearContainer {
public:
DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len)
: value(len) {
if (!value.empty()) {
DeepState_SymbolizeData(&(value.front()), &(value.back()));
}
}
DEEPSTATE_INLINE SymbolicLinearContainer(void)
: SymbolicLinearContainer(DeepState_SizeInRange(0, 32)) {}
DEEPSTATE_INLINE operator T (void) const {
return value;
}
T value;
};
template <>
class Symbolic<std::string> : public SymbolicLinearContainer<std::string> {
using SymbolicLinearContainer::SymbolicLinearContainer;
};
template <>
class Symbolic<std::wstring> : public SymbolicLinearContainer<std::wstring> {
using SymbolicLinearContainer::SymbolicLinearContainer;
};
template <typename T>
class Symbolic<std::vector<T>> :
public SymbolicLinearContainer<std::vector<T>> {};
#define MAKE_SYMBOL_SPECIALIZATION(Tname, tname) \
template <> \
class Symbolic<tname> { \
@@ -313,6 +279,40 @@ static T Pump(T val, unsigned max=10) {
return Minimize(val);
}
template <typename T>
class SymbolicLinearContainer {
public:
DEEPSTATE_INLINE explicit SymbolicLinearContainer(size_t len)
: value(len) {
if (!value.empty()) {
DeepState_SymbolizeData(&(value.front()), &(value.back()));
}
}
DEEPSTATE_INLINE SymbolicLinearContainer(void)
: SymbolicLinearContainer(Pump(DeepState_SizeInRange(0, 32), 32)) {}
DEEPSTATE_INLINE operator T (void) const {
return value;
}
T value;
};
template <>
class Symbolic<std::string> : public SymbolicLinearContainer<std::string> {
using SymbolicLinearContainer::SymbolicLinearContainer;
};
template <>
class Symbolic<std::wstring> : public SymbolicLinearContainer<std::wstring> {
using SymbolicLinearContainer::SymbolicLinearContainer;
};
template <typename T>
class Symbolic<std::vector<T>> :
public SymbolicLinearContainer<std::vector<T>> {};
template <typename... Args>
inline static void ForAll(void (*func)(Args...)) {
func(Symbolic<Args>()...);

78
tests/logrun.py
View File

@@ -4,44 +4,44 @@ import time
import sys
def logrun(cmd, file, timeout):
sys.stderr.write("\n\n" + ("=" * 80) + "\n")
sys.stderr.write("RUNNING: ")
sys.stderr.write(" ".join(cmd) + "\n\n")
sys.stderr.flush()
with open(file, 'w') as outf:
p = subprocess.Popen(cmd, stdout=outf, stderr=outf)
start = time.time()
oldContents = ""
lastOutput = time.time()
while (p.poll() is None) and ((time.time() - start) < timeout):
if (time.time() - lastOutput) > 300:
sys.stderr.write(".")
sys.stderr.flush()
lastOutput = time.time()
with open(file, 'r') as inf:
contents = inf.read()
if len(contents) > len(oldContents):
sys.stderr.write(contents[len(oldContents):])
sys.stderr.flush()
oldContents = contents
lastOutput = time.time()
time.sleep(0.05)
totalTime = time.time() - start
sys.stderr.write("\n")
rv = (p.returncode, contents)
if p.poll() is None:
rv = ("TIMEOUT", contents)
if "Traceback (most recent call last)" in contents:
rv = ("EXCEPTION RAISED", contents)
if "internal error" in contents:
rv = ("INTERNAL ERROR", contents)
sys.stderr.write("\nDONE\n\n")
sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n")
sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n")
sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n")
sys.stderr.write("=" * 80 + "\n")
sys.stderr.write("\n\n" + ("=" * 80) + "\n")
sys.stderr.write("RUNNING: ")
sys.stderr.write(" ".join(cmd) + "\n\n")
sys.stderr.flush()
with open(file, 'w') as outf:
p = subprocess.Popen(cmd, stdout=outf, stderr=outf)
start = time.time()
oldContents = ""
lastOutput = time.time()
while (p.poll() is None) and ((time.time() - start) < timeout):
if (time.time() - lastOutput) > 300:
sys.stderr.write(".")
sys.stderr.flush()
lastOutput = time.time()
with open(file, 'r') as inf:
contents = inf.read()
if len(contents) > len(oldContents):
sys.stderr.write(contents[len(oldContents):])
sys.stderr.flush()
oldContents = contents
lastOutput = time.time()
time.sleep(0.05)
totalTime = time.time() - start
sys.stderr.write("\n")
rv = (p.returncode, contents)
if p.poll() is None:
rv = ("TIMEOUT", contents)
if "Traceback (most recent call last)" in contents:
rv = ("EXCEPTION RAISED", contents)
if "internal error" in contents:
rv = ("INTERNAL ERROR", contents)
sys.stderr.write("\nDONE\n\n")
sys.stderr.write("TOTAL EXECUTION TIME: " + str(totalTime) + "\n")
sys.stderr.write("RETURN VALUE: " + str(p.returncode) + "\n")
sys.stderr.write("RETURNING AS RESULT: " + str(rv[0]) + "\n")
sys.stderr.write("=" * 80 + "\n")
return rv
return rv

224
tests/test_basic_functionality.py
View File

@@ -4,129 +4,145 @@ from unittest import TestCase
import logrun
class TestBasicFunctionality(TestCase):
def test_basic_functionality(self):
if os.getenv("DEEPSTATE_CMD") is not None:
deepstates = [os.getenv("DEEPSTATE_CMD")]
else:
deepstates = ["deepstate-angr", "deepstate-manticore"]
class DeepStateTestCase(TestCase):
def test_angr(self):
self.run_test(self, "deepstate-angr")
print("RUNNING WITH DEEPSTATE COMMANDS:", deepstates)
def test_manticore(self):
self.run_test(self, "deepstate-manticore")
for deepstate in deepstates:
if os.getenv("TASK") is None or os.getenv("TASK") == "CRASH":
(r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Passed: Crash_SegFault" in output)
foundCrashSave = False
for line in output.split("\n"):
if ("Saving input to" in line) and (".crash" in line):
foundCrashSave = True
self.assertTrue(foundCrashSave)
class CrashTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/Crash"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
if os.getenv("TASK") is None or os.getenv("TASK") == "KLEE":
(r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Passed: Crash_SegFault" in output)
foundCrashSave = False
for line in output.split("\n"):
if ("Saving input to" in line) and (".crash" in line):
foundCrashSave = True
self.assertTrue(foundCrashSave)
self.assertTrue("zero" in output)
self.assertTrue("positive" in output)
self.assertTrue("negative" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "PRIMES":
(r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
class KleeTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/Klee", "--klee"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
self.assertTrue("zero" in output)
self.assertTrue("positive" in output)
self.assertTrue("negative" in output)
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "TAKEOVER":
(r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("hi" in output)
self.assertTrue("bye" in output)
self.assertTrue("was not greater than" in output)
class PrimesTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/Primes"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
foundPassSave = False
for line in output.split("\n"):
if ("Saving input to" in line) and (".pass" in line):
foundPassSave = True
self.assertTrue(foundPassSave)
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes" in output)
self.assertTrue("Failed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "FIXTURE":
(r, output) = logrun.logrun([deepstate, "build/examples/Fixture"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes" in output)
self.assertTrue("Passed: PrimePolynomial_OnlyGeneratesPrimes_NoStreaming" in output)
self.assertTrue("Passed: MyTest_Something" in output)
self.assertFalse("Failed: MyTest_Something" in output)
self.assertTrue("Setting up!" in output)
self.assertTrue("Tearing down!" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "LISTS":
(r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
"deepstate.out", 3000)
self.assertEqual(r, 0)
class TakeOverTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/TakeOver", "--take_over"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Passed: Vector_DoubleReversal" in output)
self.assertFalse("Failed: Vector_DoubleReversal" in output)
self.assertTrue("hi" in output)
self.assertTrue("bye" in output)
self.assertTrue("was not greater than" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "STREAMINGANDFORMATTING":
(r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"],
"deepstate.out", 1800)
#self.assertEqual(r, 0)
foundPassSave = False
for line in output.split("\n"):
if ("Saving input to" in line) and (".pass" in line):
foundPassSave = True
self.assertTrue(foundPassSave)
self.assertTrue("Failed: Streaming_BasicLevels" in output)
self.assertTrue("This is a debug message" in output)
self.assertTrue("This is an info message" in output)
self.assertTrue("This is a warning message" in output)
self.assertTrue("This is a error message" in output)
self.assertTrue("This is a info message again" in output)
self.assertTrue(": 97" in output)
self.assertTrue(": 1" in output)
self.assertTrue(": 1.000000" in output)
self.assertTrue(": string" in output)
self.assertTrue("hello string=world" in output)
self.assertTrue("hello again!" in output)
self.assertTrue("Passed: Formatting_OverridePrintf" in output)
self.assertFalse("Failed: Formatting_OverridePrintf" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "ONEOF":
(r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
class FixtureTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/Fixture"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
self.assertTrue("Passed: MyTest_Something" in output)
self.assertFalse("Failed: MyTest_Something" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "OVERFLOW":
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Setting up!" in output)
self.assertTrue("Tearing down!" in output)
self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output)
self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output)
self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output)
self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output)
if os.getenv("TASK") is None or os.getenv("TASK") == "ARITHMETIC":
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)
self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)
self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)
class ListsTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/Lists"],
"deepstate.out", 3000)
self.assertEqual(r, 0)
self.assertTrue("Passed: Vector_DoubleReversal" in output)
self.assertFalse("Failed: Vector_DoubleReversal" in output)
class StreamingAndFormattingTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/StreamingAndFormatting"],
"deepstate.out", 1800)
#self.assertEqual(r, 0)
self.assertTrue("Failed: Streaming_BasicLevels" in output)
self.assertTrue("This is a debug message" in output)
self.assertTrue("This is an info message" in output)
self.assertTrue("This is a warning message" in output)
self.assertTrue("This is a error message" in output)
self.assertTrue("This is a info message again" in output)
self.assertTrue(": 97" in output)
self.assertTrue(": 1" in output)
self.assertTrue(": 1.000000" in output)
self.assertTrue(": string" in output)
self.assertTrue("hello string=world" in output)
self.assertTrue("hello again!" in output)
self.assertTrue("Passed: Formatting_OverridePrintf" in output)
self.assertFalse("Failed: Formatting_OverridePrintf" in output)
class OneOfTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/OneOf"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Failed: OneOfExample_ProduceSixtyOrHigher" in output)
self.assertTrue("Passed: OneOfExample_ProduceSixtyOrHigher" in output)
class OverflowTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerOverflow"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Failed: SignedInteger_AdditionOverflow" in output)
self.assertTrue("Passed: SignedInteger_AdditionOverflow" in output)
self.assertTrue("Failed: SignedInteger_MultiplicationOverflow" in output)
self.assertTrue("Passed: SignedInteger_MultiplicationOverflow" in output)
class ArithmeticTest(DeepStateTestCase):
def run_test(self, deepstate):
(r, output) = logrun.logrun([deepstate, "build/examples/IntegerArithmetic", "--num_workers", "4"],
"deepstate.out", 1800)
self.assertEqual(r, 0)
self.assertTrue("Failed: Arithmetic_InvertibleMultiplication_CanFail" in output)
self.assertTrue("Passed: Arithmetic_AdditionIsCommutative" in output)
self.assertFalse("Failed: Arithmetic_AdditionIsCommutative" in output)
self.assertTrue("Passed: Arithmetic_AdditionIsAssociative" in output)
self.assertFalse("Failed: Arithmetic_AdditionIsAssociative" in output)
self.assertTrue("Passed: Arithmetic_InvertibleMultiplication_CanFail" in output)