Remove md5 tests and replace with set comparison (#374)

* remove md5 tests and replace with comparison

* fix directory path issue

* remove exact check as test is not deterministic

* fix argument assertion test and remove cdet_visited.txt

* allow extra addresses in actual to account for inter-machine differences

manticore/manticore.py

@@ -862,7 +862,7 @@ class Manticore(object):
         '''
         self._executor.shutdown()
 
-    def _assertions_callback(self, state):
+    def _assertions_callback(self, state, instruction):
         pc = state.cpu.PC
         if pc not in self._assertions:
             return
@@ -873,7 +873,7 @@ class Manticore(object):
 
         #This will interpret the buffer specification written in INTEL ASM.
         # (It may dereference pointers)
-        assertion = parse(program, state.cpu.read, state.cpu.read_register)
+        assertion = parse(program, state.cpu.read_int, state.cpu.read_register)
         if not solver.can_be_true(state.constraints, assertion):
             logger.info(str(state.cpu))
             logger.info("Assertion %x -> {%s} does not hold. Aborting state.",

tests/test_binaries.py

@@ -22,6 +22,15 @@ class IntegrationTest(unittest.TestCase):
         # Remove the directory after the test
         shutil.rmtree(self.test_dir)
 
+    def _loadVisitedSet(self, visited):
+
+        self.assertTrue(os.path.exists(visited))
+        vitems = open(visited, 'r').read().splitlines()
+
+        vitems = map(lambda x: int(x[2:], 16), vitems)
+
+        return set(vitems)
+
     def _runWithTimeout(self, procargs, logfile, timeout=1200):
 
         with open(os.path.join(os.pardir, logfile), "w") as output:
@@ -43,8 +52,6 @@ class IntegrationTest(unittest.TestCase):
         self.assertTrue(filename.startswith(os.getcwd()))
         filename = filename[len(os.getcwd())+1:]
         data = file(filename,'rb').read()
-        self.assertEqual(len(data), 767152)
-        self.assertEqual(hashlib.md5(data).hexdigest() , '00fb23e47831a1054ca4a74656035472')
         workspace = '%s/workspace'%self.test_dir
         t = time.time()
         with open(os.path.join(os.pardir, '%s/output.log'%self.test_dir), "w") as output:
@@ -66,30 +73,25 @@ class IntegrationTest(unittest.TestCase):
         output = subprocess.check_output(['python', '-m', 'manticore', filename])
         self.assertLessEqual(len(output.splitlines()), 25)
 
-    @unittest.skip('TODO(mark); skipping so we can move on with our lives and merge x86_new. ask felipe to fix later.')
     def testArgumentsAssertions(self):
         dirname = os.path.dirname(__file__)
         filename = os.path.abspath(os.path.join(dirname, 'binaries/arguments_linux_amd64'))
         self.assertTrue(filename.startswith(os.getcwd()))
         filename = filename[len(os.getcwd())+1:]
-        SE = os.path.join(dirname, '../main.py')
         data = file(filename,'rb').read()
-        self.assertEqual(len(data), 767152)
-        self.assertEqual(hashlib.md5(data).hexdigest() , '00fb23e47831a1054ca4a74656035472')
         workspace = '%s/workspace'%self.test_dir
         assertions = '%s/assertions.txt'%self.test_dir
         file(assertions,'w').write('0x0000000000401003 ZF == 1')
         with open(os.path.join(os.pardir, '%s/output.log'%self.test_dir), "w") as output:
-            self._runWithTimeout(['python', SE,
-                                '--workspace', workspace,
-                                '--proc', '4',
-                                '--assertions', assertions,
-                                filename,
-                                '+++++++++'], stdout=output)
-        data = file('%s/visited.txt'%workspace,'r').read()
-        data = '\n'.join(sorted(set(data.split('\n'))))
-        self.assertEqual(hashlib.md5(data).hexdigest() , 'c52d7d471ba5c94fcf59936086821a6b')
-
+            subprocess.check_call(['python', '-m', 'manticore',
+                                   '--workspace', workspace,
+                                   '--proc', '4',
+                                   '--assertions', assertions,
+                                   filename,
+                                   '+++++++++'], stdout=output)
+        actual = self._loadVisitedSet(os.path.join(dirname, '%s/visited.txt'%workspace))
+        expected = self._loadVisitedSet(os.path.join(dirname, 'reference/arguments_linux_arm64_visited.txt'))
+        self.assertGreaterEqual(actual, expected)
 
     def testDecree(self):
         dirname = os.path.dirname(__file__)
@@ -98,8 +100,6 @@ class IntegrationTest(unittest.TestCase):
         filename = filename[len(os.getcwd())+1:]
         SE = os.path.join(dirname, '../main.py')
         data = file(filename,'rb').read()
-        self.assertEqual(len(data), 1828)
-        self.assertEqual(hashlib.md5(data).hexdigest() , '8955a29d51c1edd39b0e53794ebcf464')
         workspace = '%s/workspace'%self.test_dir
         self._runWithTimeout(['python', '-m', 'manticore',
                     '--workspace', workspace,
@@ -108,9 +108,8 @@ class IntegrationTest(unittest.TestCase):
                     '--policy', 'uncovered',
                     filename], '%s/output.log'%self.test_dir)
 
-        data = file('%s/visited.txt'%workspace,'r').read()
-        visited = len(set(data.split('\n')))
-        self.assertTrue(visited > 100 )
+        actual = self._loadVisitedSet(os.path.join(dirname, '%s/visited.txt'%workspace))
+        self.assertTrue(len(actual) > 100 )
 
 
 if __name__ == '__main__':