diff --git a/chatops/python/validate_report.py b/chatops/python/validate_report.py index 6c182f7..a0ec428 100644 --- a/chatops/python/validate_report.py +++ b/chatops/python/validate_report.py @@ -45,8 +45,8 @@ SNIPPETDIR = 'snippets/' TEMPLATEDIR = 'templates/' OFFERTE = '/offerte.xml' REPORT = '/report.xml' -WARN_LINE = 100 # There should be a separation character after x characters... -MAX_LINE = 130 # ... and before y +WARN_LINE = 80 # There should be a separation character after x characters... +MAX_LINE = 86 # ... and before y if DOCBUILDER: @@ -63,8 +63,8 @@ def parse_arguments(): Parses command line arguments. """ parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent('''\ + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent('''\ validate_report - validates offer letters and reports Copyright (C) 2015-2016 Radically Open Security (Peter Mosmans) @@ -177,7 +177,7 @@ def validate_files(filenames, options): if (OFFERTE in filename and options['offer']) or \ (REPORT in filename and not options['no_report']): masters.append(filename) - # try: + # try: type_result, xml_type = validate_xml(filename, options) result = result and type_result if 'non-finding' in xml_type: @@ -221,13 +221,16 @@ def validate_xml(filename, options): """ result = True xml_type = '' + # crude check whether the file is outside the pentext framework + if 'notes' in filename: + return result, xml_type print_output(options, 'Validating XML file: {0}'.format(filename)) try: with open(filename, 'rb') as xml_file: xml.sax.parse(xml_file, xml.sax.ContentHandler()) - tree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False)) - type_result, xml_type = validate_type(tree, filename, options) - result = validate_long_lines(tree, filename, options) and result and type_result + tree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False)) + type_result, xml_type = validate_type(tree, filename, options) + result = validate_long_lines(tree, filename, options) and result and type_result if options['edit'] and not result: open_editor(filename) except (xml.sax.SAXException, ElementTree.ParseError) as exception: @@ -310,7 +313,8 @@ def validate_type(tree, filename, options): else: if attribute == 'threatLevel' and root.attrib[attribute] not in \ ('Low', 'Moderate', 'Elevated', 'High', 'Extreme'): - print('[-] threatLevel is not Low, Moderate, High, Elevated or Extreme: {0}'.format(root.attrib[attribute])) + print('[-] threatLevel is not Low, Moderate, High, Elevated or Extreme: {0}'. + format(root.attrib[attribute])) result = False if attribute == 'type' and (options['capitalization'] and not \ is_capitalized(root.attrib[attribute])): @@ -351,7 +355,7 @@ def validate_type(tree, filename, options): def validate_long_lines(tree, filename, options): """ - Checks whether 
 section contains lines longer than MAX_LINE characters
+    Checks whether pre or code section contains lines longer than MAX_LINE characters
     Returns True if the file validated successfully.
     """
     if not options['long']:
@@ -359,29 +363,31 @@ def validate_long_lines(tree, filename, options):
     result = True
     fix = False
     root = tree.getroot()
-    for pre_section in root.iter('pre'):
+    for pre_section in [j for section in ('pre', 'code') for j in root.iter(section)]:
         if pre_section.text:
             fixed_text = ''
             for line in pre_section.text.splitlines():
-                fixed_line = line
-                if len(line.strip()) > MAX_LINE:
-                    if ' ' not in line[WARN_LINE:MAX_LINE]:
-                        print('[-] {0} Line inside 
 too long: {1}'.
-                              format(filename, line.encode('utf-8')[WARN_LINE:]))
-                        result = False
-                        for split in ['"', '\'', '=', '-', ';']:
-                            if split in line.encode('utf-8').strip()[WARN_LINE:MAX_LINE]:
-                                print('[A] can be fixed')
-                                fix = True
-                                index = line.find(split, WARN_LINE)
-                                fixed_line = line[:index + 1] + '\n'
-                                fixed_line += line[index + 1:]
-                fixed_text += fixed_line.encode('utf-8')
-    if fix:
-        if options['auto_fix']:
-            print('[+] Automatically fixed {0}'.format(filename))
-#            tree.write(filename)
-        print(fixed_text)
+                while len(line) > MAX_LINE:
+                    result = False
+                    print('[-] {0} Line inside {1} too long: {2}'.
+                          format(filename, section, line.encode('utf-8')[MAX_LINE:]))
+                    cutpoint = MAX_LINE
+                    for split in [' ', '"', '\'', '=', '-', ';']:
+                        if split in line.encode('utf-8')[WARN_LINE:MAX_LINE]:
+                            cutpoint = line.find(split, WARN_LINE, MAX_LINE)
+                    fix = True
+                    fixed_line = line[:cutpoint] + '\n'
+                    print('cutted line {0}'.format(line))
+                    line = line[cutpoint:]
+                    fixed_text += fixed_line.encode('utf-8')
+                    print('[A] can be fixed (breaking at {0}): {1}'.format(cutpoint, fixed_line))
+                fixed_text += line + '\n'
+            if fix and options['auto_fix']:
+                print('[+] Automatically fixed {0}'.format(filename))
+                pre_section.text = fixed_text
+                print(fixed_text)
+                tree.write(filename)
+                close_file(filename)
     return result
 
 
@@ -392,14 +398,14 @@ def validate_master(filename, findings, non_findings, scans, options):
     result = True
     include_findings = []
     include_nonfindings = []
-    print_output(options, '[*] Validating master file {0}'.format(filename))
+    print_output(options, 'Validating master file {0}'.format(filename))
     try:
         xmltree = ElementTree.parse(filename,
                                     ElementTree.XMLParser(strip_cdata=False))
         if not find_keyword(xmltree, 'TODO', filename):
             print('[-] Keyword checks failed for {0}'.format(filename))
             result = False
-        print_output(options, 'Performing cross check on findings, non-findings and scans...')
+            print_output(options, 'Performing cross check on findings, non-findings and scans...')
         for finding in findings:
             if not cross_check_file(filename, finding):
                 print('[A] Cross check failed for finding {0}'.
@@ -413,7 +419,7 @@ def validate_master(filename, findings, non_findings, scans, options):
                 include_nonfindings.append(non_finding)
                 result = False
         if result:
-            print_output(options, '[+] Cross checks successful')
+            print_output(options, 'Cross checks successful')
     except (ElementTree.ParseError, IOError) as exception:
         print('[-] validating {0} failed ({1})'.format(filename, exception))
         result = False
@@ -450,7 +456,7 @@ def cross_check_file(filename, external):
         print('[-] could not find a reference in {0} to {1}'.format(filename, external))
         result = False
     return result
-    
+
 
 def add_include(filename, identifier, findings):
     """
@@ -465,18 +471,19 @@ def add_include(filename, identifier, findings):
         for finding in findings:
             new_finding = ElementTree.XML(''.format(finding))
             finding_section.append(new_finding)
-        tree.write(filename, encoding="utf-8", xml_declaration=True, pretty_print=True)
+            tree.write(filename, encoding="utf-8", xml_declaration=True, pretty_print=True)
+
 
 def close_file(filename):
     """
     Replace placeholder with proper XML include.
     """
-    f = open(filename,'r')
+    f = open(filename, 'r')
     filedata = f.read()
     f.close()
-    newdata = filedata.replace("placeholderinclude","xi:include")
+    newdata = filedata.replace("placeholderinclude", "xi:include")
     fileout = filename
-    f = open(fileout,'w')
+    f = open(fileout, 'w')
     f.write(newdata)
     f.close()
     tree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False))
@@ -513,11 +520,11 @@ def main():
         options['long'] = True
     if options['learn']:
         print_output(options, 'Adding unknown words to {0}'.format(VOCABULARY))
-#    if options['spelling']:
-#        if not os.path.exists(VOCABULARY):
-#            print_output(options, 'Creating project-specific vocabulary file {0}'.
-#                  format(VOCABULARY))
-#            options['learn'] = True
+        #    if options['spelling']:
+        #        if not os.path.exists(VOCABULARY):
+        #            print_output(options, 'Creating project-specific vocabulary file {0}'.
+        #                  format(VOCABULARY))
+        #            options['learn'] = True
     print_output(options, 'Validating all XML files...')
     result = validate_files(all_files(), options)
     if result: