Merge branch 'master' of https://github.com/radicallyopensecurity/pentext into development

# Conflicts:
#	chatops/bash/handler_build

chatops/bash/handler_build

@@ -120,6 +120,7 @@ build() {
 add_to_repo() {
     git add $TARGETPDF
     git add target/waiver_?*.pdf &>/dev/null
+    git add target/execsummary.pdf &>/dev/null
     git commit -q -m "$TARGETPDF proudly manufactured using ChatOps" &>/dev/null
     git push -q >/dev/null
 }

chatops/bash/releaser.sh

@@ -42,7 +42,7 @@ if [ -f ${source} ]; then
     else
         cp -v ${source} ${fullname}
         PASS=$(head -c 25 /dev/random | base64 | head -c 25)
-        zip --password ${PASS} "${fullname}.zip" ${fullname} 2>/dev/null && echo "Zip file encrypted with password '${PASS}'"
+        7z a -p${PASS} "${fullname}.zip" ${fullname} 2>/dev/null && echo "Zip file encrypted with password '${PASS}'"
     fi
 else
     echo "Could not find source ${source}"

chatops/python/docbuilder.py

@@ -32,6 +32,7 @@ import textwrap
 GITREV = 'GITREV'  # Magic tag which gets replaced by the git short commit hash
 OFFERTE = 'generate_offerte.xsl'  # XSL for generating waivers
 WAIVER = 'waiver_'  # prefix for waivers
+EXECSUMMARY = 'execsummary' # generating an executive summary instead of a report
 
 
 def parse_arguments():
@@ -55,6 +56,9 @@ the Free Software Foundation, either version 3 of the License, or
                         help='overwrite output file if it already exists')
     parser.add_argument('-date', action='store',
                         help='the invoice date')
+    parser.add_argument('-execsummary', action='store',
+                        help="""create an executive summary as well as a report (true/false). 
+                        Default: false """)
     parser.add_argument('--fop-config', action='store',
                         default='/etc/docbuilder/rosfop.xconf',
                         help="""fop configuration file (default
@@ -141,6 +145,8 @@ def to_fo(options):
         cmd.append('INVOICE_NO=' + options['invoice'])
     if options['date']:
         cmd.append('DATE=' + options['date'])
+    if options['execsummary']:
+        cmd.append('EXEC_SUMMARY=' + options['execsummary'])
     process = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE)
     stdout, stderr = process.communicate()
     print_output(stdout, stderr)
@@ -222,6 +228,23 @@ def main():
             except OSError as exception:
                 print_exit('[-] ERR: {0}'.format(exception.strerror),
                            exception.errno)
+        if options['execsummary'] == 'true':  # we're generating a summary as well as a report
+            report_output = options['output']
+            verboseprint('generating additional executive summary')
+            output_dir = os.path.dirname(options['output'])
+            fop_dir = os.path.dirname(options['fop'])
+            try:
+                for fop in [os.path.splitext(x)[0] for x in
+                            os.listdir(fop_dir) if x.endswith('fo')]:
+                    if EXECSUMMARY in fop:
+                        options['output'] = output_dir + os.sep + fop + '.pdf'
+                    else:
+                        options['output'] = report_output
+                    options['fop'] = fop_dir + os.sep + fop + '.fo'
+                    result = to_pdf(options) and result
+            except OSError as exception:
+                print_exit('[-] ERR: {0}'.format(exception.strerror),
+                           exception.errno)
         else:
             result = to_pdf(options)
 

chatops/python/gitlab-to-pentext.py

@@ -146,7 +146,8 @@ def list_issues(gitserver, options):
     Lists all issues for options['issues']
     """
     try:
-        for issue in gitserver.project_issues.list(project_id=options['issues']):
+        for issue in gitserver.project_issues.list(project_id=options['issues'],
+                                                   per_page=99):
             if issue.state == 'closed' and not options['closed']:
                 continue
             if 'finding' in issue.labels:
@@ -251,7 +252,7 @@ def valid_filename(filename):
     """
     result = ''
     for char in filename.strip():
-        if char in [':', '/', '.', '\\', ' ', '[', ']', '(', ')', '\'']:
+        if char in ['*', ':', '/', '.', '\\', ' ', '[', ']', '(', ')', '\'']:
             if len(char) and not result.endswith('-'):
                 result += '-'
         else:

chatops/python/validate_report.py

@@ -23,6 +23,7 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 import argparse
+import logging
 import mmap
 import os
 import re
@@ -46,8 +47,10 @@ VOCABULARY = 'project-vocabulary.pws'
 # Snippets may contain XML fragments without the proper entities
 EXAMPLEDIR = 'examples/'
 NOT_CAPITALIZED = ['a', 'an', 'and', 'as', 'at', 'but', 'by', 'for', 'in',
-                   'nor', 'of', 'on', 'or', 'the', 'to', 'up']
+                   'jQuery', 'jQuery-UI', 'nor', 'of', 'on', 'or', 'the', 'to',
+                   'up']
 SNIPPETDIR = 'snippets/'
+STATUS = 25 # loglevel for 'generic' status messages
 TEMPLATEDIR = 'templates/'
 OFFERTE = '/offerte.xml'
 REPORT = '/report.xml'
@@ -60,8 +63,29 @@ if DOCBUILDER:
     import proxy_vagrant
 try:
     import aspell
-except:
+except ImportError:
-    print('[-] aspell not installed: spelling not available')
+    print('[-] aspell not installed: spelling not available',)
+
+
+class LogFormatter(logging.Formatter):
+    """
+    Format log messages according to their type.
+    """
+    # DEBUG   = (10) debug status messages
+    # INFO    = (20) verbose status messages
+    # STATUS  = (25) generic status messages
+    # WARNING = (30) warning messages (= errors in validation)
+    # ERROR   = (40) error messages (= program errors)
+    FORMATS = {logging.DEBUG :"DEBUG: %(module)s: %(lineno)d: %(message)s",
+               logging.INFO : "[*] %(message)s",
+               STATUS : "[+] %(message)s",
+               logging.WARN : "[-] %(message)s",
+               logging.ERROR : "ERROR: %(message)s",
+               'DEFAULT' : "%(message)s"}
+
+    def format(self, record):
+        self._fmt = self.FORMATS.get(record.levelno, self.FORMATS['DEFAULT'])
+        return logging.Formatter.format(self, record)
 
 
 def parse_arguments():
@@ -106,21 +130,37 @@ the Free Software Foundation, either version 3 of the License, or
     return vars(parser.parse_args())
 
 
-def validate_spelling(tree, filename, options):
+def initialize_speller():
     """
-    Checks spelling of text within tags.
+    Initialize and return speller module.
-    If options['learn'], then unknown words will be added to the dictionary.
     """
-    result = True
+    speller = None
     try:
         speller = aspell.Speller(('lang', 'en'),
                                  ('personal-dir', '.'),
                                  ('personal', VOCABULARY))
-    except:  # some versions of aspell use a different path
+    except aspell.AspellConfigError as exception:  # some versions of aspell use a different path
-        speller = aspell.Speller(('lang', 'en'),
+        logging.debug('Encountered exception when trying to intialize spelling: %s',
-                                 ('personal-path', './' + VOCABULARY))
+                      exception)
-    if options['debug']:
+        try:
-        [print(i[0] + ' ' + str(i[2]) + '\n') for i in speller.ConfigKeys()]
+            speller = aspell.Speller(('lang', 'en'),
+                                     ('personal-path', './' + VOCABULARY))
+        except aspell.AspellSpellerError as exception:
+            logging.error('Could not initialize speller: %s', exception)
+    if speller:
+        [logging.debug('%s %s', i[0], i[2]) for i in speller.ConfigKeys()]
+    return speller
+
+
+def validate_spelling(tree, filename, options, speller):
+    """
+    Check spelling of text within tags.
+    If options['learn'], then unknown words will be added to the dictionary.
+    """
+    result = True
+    if not speller:
+        options['spelling'] = False
+        return result
     try:
         root = tree.getroot()
         for section in root.iter():
@@ -132,12 +172,13 @@ def validate_spelling(tree, filename, options):
                             speller.addtoPersonal(word)
                         else:
                             result = False
-                            print('[-] Misspelled (unknown) word {0} in {1}'.
+                            logging.warning('Misspelled (unknown) word %s in %s',
-                                  format(word.encode('utf-8'), filename))
+                                            word.encode('utf-8'), filename)
         if options['learn']:
             speller.saveAllwords()
     except aspell.AspellSpellerError as exception:
-        print('[-] Spelling disabled ({0})'.format(exception))
+        logging.error('Disabled spelling (%s)', exception)
+        options['spelling'] = False
     return result
 
 
@@ -152,6 +193,9 @@ def all_files():
 
 
 def open_editor(filename):
+    """
+    Open editor with file to edit.
+    """
     if sys.platform in ('linux', 'linux2'):
         editor = os.getenv('EDITOR')
         if editor:
@@ -176,6 +220,7 @@ def validate_files(filenames, options):
     findings = []
     non_findings = []
     scans = []
+    speller = initialize_speller()
     for filename in filenames:
         if (filename.lower().endswith('.xml') or
                 filename.lower().endswith('xml"')):
@@ -184,7 +229,7 @@ def validate_files(filenames, options):
                    (REPORT in filename and not options['no_report']):
                     masters.append(filename)
                     # try:
-                type_result, xml_type = validate_xml(filename, options)
+                type_result, xml_type = validate_xml(filename, options, speller)
                 result = result and type_result
                 if 'non-finding' in xml_type:
                     non_findings.append(filename)
@@ -200,16 +245,6 @@ def validate_files(filenames, options):
     return result
 
 
-def print_output(options, stdout, stderr=None):
-    """
-    Prints out standard out and standard err using the verboseprint function.
-    """
-    if stdout and options['verbose']:
-        print('[+] {0}'.format(stdout))
-    if stderr and options['verbose']:
-        print('[-] {0}'.format(stderr))
-
-
 def validate_report():
     """
     Validates XML report file by trying to build it.
@@ -220,7 +255,7 @@ def validate_report():
     return proxy_vagrant.execute_command(host, command)
 
 
-def validate_xml(filename, options):
+def validate_xml(filename, options, speller):
     """
     Validates XML file by trying to parse it.
     Returns True if the file validated successfully.
@@ -230,12 +265,12 @@ def validate_xml(filename, options):
     # crude check whether the file is outside the pentext framework
     if 'notes' in filename:
         return result, xml_type
-    print_output(options, 'Validating XML file: {0}'.format(filename))
+    logging.info('Validating XML file: %s', filename)
     try:
         with open(filename, 'rb') as xml_file:
             xml.sax.parse(xml_file, xml.sax.ContentHandler())
             tree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False))
-            type_result, xml_type = validate_type(tree, filename, options)
+            type_result, xml_type = validate_type(tree, filename, options, speller)
             result = validate_long_lines(tree, filename, options) and result and type_result
         if options['edit'] and not result:
             open_editor(filename)
@@ -282,7 +317,7 @@ def capitalize(line):
     return capitalized.strip()
 
 
-def validate_type(tree, filename, options):
+def validate_type(tree, filename, options, speller):
     """
     Performs specific checks based on type.
     Currently only finding and non-finding are supported.
@@ -294,7 +329,7 @@ def validate_type(tree, filename, options):
     attributes = []
     tags = []
     if options['spelling']:
-        result = validate_spelling(tree, filename, options)
+        result = validate_spelling(tree, filename, options, speller)
     if xml_type == 'pentest_report':
         attributes = ['findingCode']
     if xml_type == 'finding':
@@ -331,11 +366,11 @@ def validate_type(tree, filename, options):
                 fix = True
     for tag in tags:
         if root.find(tag) is None:
-            print('[-] Missing tag in {0}: {1}'.format(filename, tag))
+            logging.warning('Missing tag in %s: %s', filename, tag)
             result = False
             continue
         if not get_all_text(root.find(tag)):
-            print('[-] Empty tag in {0}: {1}'.format(filename, tag))
+            logging.warning('Empty tag in %s: %s', filename, tag)
             result = False
             continue
         if tag == 'title' and (options['capitalization'] and \
@@ -404,14 +439,14 @@ def validate_master(filename, findings, non_findings, scans, options):
     result = True
     include_findings = []
     include_nonfindings = []
-    print_output(options, 'Validating master file {0}'.format(filename))
+    logging.info('Validating master file %s', filename)
     try:
         xmltree = ElementTree.parse(filename,
                                     ElementTree.XMLParser(strip_cdata=False))
         if not find_keyword(xmltree, 'TODO', filename):
             print('[-] Keyword checks failed for {0}'.format(filename))
             result = False
-            print_output(options, 'Performing cross check on findings, non-findings and scans...')
+            logging.info('Performing cross check on findings, non-findings and scans...')
         for finding in findings:
             if not cross_check_file(filename, finding):
                 print('[A] Cross check failed for finding {0}'.
@@ -420,23 +455,22 @@ def validate_master(filename, findings, non_findings, scans, options):
                 result = False
         for non_finding in non_findings:
             if not cross_check_file(filename, non_finding):
-                print('[A] Cross check failed for non-finding {0}'.
+                logging.warning('Cross check failed for non-finding %s', non_finding)
-                      format(non_finding))
                 include_nonfindings.append(non_finding)
                 result = False
         if result:
-            print_output(options, 'Cross checks successful')
+            logging.info('Cross checks successful')
     except (ElementTree.ParseError, IOError) as exception:
-        print('[-] validating {0} failed ({1})'.format(filename, exception))
+        logging.warning('Validating %s failed: %s', filename, exception)
         result = False
     if not result:
         if options['auto_fix']:
             add_include(filename, 'findings', include_findings)
             add_include(filename, 'nonFindings', include_nonfindings)
             close_file(filename)
-            print('[+] Automatically fixed {0}'.format(filename))
+            logging.info('Automatically fixed %s', filename)
         else:
-            print('[+] NOTE: Items with [A] can be fixed automatically, use --auto-fix')
+            logging.warning('Item can be fixed automatically, use --auto-fix')
     return result
 
 
@@ -448,7 +482,7 @@ def report_string(report_file):
         report = open(report_file)
         return mmap.mmap(report.fileno(), 0, access=mmap.ACCESS_READ)
     except IOError as exception:
-        print('[-] Could not open {0} ({1})'.format(report_file, exception))
+        logging.critical('Could not open %s: %s', report_file, exception)
         sys.exit(-1)
 
 
@@ -459,7 +493,7 @@ def cross_check_file(filename, external):
     result = True
     report_text = report_string(filename)
     if report_text.find(external) == -1:
-        print('[-] could not find a reference in {0} to {1}'.format(filename, external))
+        logging.warning('Could not find a reference in %s to %s', filename, external)
         result = False
     return result
 
@@ -507,11 +541,29 @@ def find_keyword(xmltree, keyword, filename):
             section = 'in {0}'.format(tag.attrib['id'])
         if tag.text:
             if keyword in tag.text:
-                print('[-] {0} found in {1} {2}'.format(keyword, filename, section))
+                logging.warning('%s found in %s %s', keyword, filename, section)
                 result = False
     return result
 
 
+def setup_logging(options):
+    """
+    Set up loghandlers according to options.
+    """
+    logger = logging.getLogger()
+    logger.setLevel(0)
+    console = logging.StreamHandler(stream=sys.stdout)
+    console.setFormatter(LogFormatter())
+    if options['debug']:
+        console.setLevel(logging.DEBUG)
+    else:
+        if options['verbose']:
+            console.setLevel(logging.INFO)
+        else:
+            logger.setLevel(STATUS)
+    logger.addHandler(console)
+
+
 def main():
     """
     The main program. Cross-checks, validates XML files and report.
@@ -521,30 +573,24 @@ def main():
     reload(sys)
     sys.setdefaultencoding('utf-8')
     options = parse_arguments()
+    setup_logging(options)
     if options['all']:
         options['capitalization'] = True
         options['long'] = True
     if options['learn']:
-        print_output(options, 'Adding unknown words to {0}'.format(VOCABULARY))
+        logging.debug('Adding unknown words to %s', VOCABULARY)
-        #    if options['spelling']:
+    logging.info('Validating all XML files...')
-        #        if not os.path.exists(VOCABULARY):
-        #            print_output(options, 'Creating project-specific vocabulary file {0}'.
-        #                  format(VOCABULARY))
-        #            options['learn'] = True
-    print_output(options, 'Validating all XML files...')
     result = validate_files(all_files(), options)
     if result:
-        print_output(options, 'Validation checks successful')
         if DOCBUILDER:
-            print_output(options, 'Validating report build...')
+            logging.info('Validating report build...')
             result = validate_report() and result
     if result:
-        print('[+] Succesfully validated everything. Good to go')
+        logging.log(STATUS, 'Validation checks successful. Good to go')
     else:
-        print('[-] Errors occurred')
+        logging.warning('Validation failed')
     if options['spelling'] and options['learn']:
-        print('[*] Don\'t forget to check the vocabulary file {0}'.
+        logging.log(STATUS('Don\'t forget to check the vocabulary file %s', VOCABULARY))
-              format(VOCABULARY))
 
 
 if __name__ == "__main__":

scripts/gitlab-to-pentext.py

@@ -119,12 +119,11 @@ def convert_markdown(text):
     Replace markdown monospace with monospace tags
     """
     result = text
-    return result
+    return result  # currently not implemented
     print('EXAMINING ' + text + ' END')
     monospace = re.findall("\`\`\`(.*?)\`\`\`", text, re.DOTALL)
     print(monospace)
     if len(monospace):
-        print('YESSS ' + monospace)
         result = {}
         result['monospace'] = ''.join(monospace)
 
@@ -158,7 +157,7 @@ def parse_arguments():
     parser = argparse.ArgumentParser(
             formatter_class=argparse.RawDescriptionHelpFormatter,
             description=textwrap.dedent('''\
-gitlab-to-pentext - imports and updates gitlab issues into PetText (XML) format
+gitlab-to-pentext - imports and updates gitlab issues into PenText (XML) format
 
             Copyright (C) 2016  Peter Mosmans [Radically Open Security]]
 This program is free software: you can redistribute it and/or modify
@@ -184,7 +183,7 @@ the Free Software Foundation, either version 3 of the License, or
     return vars(parser.parse_args())
 
 
-def preflight_checks(options):
+def preflight_checks():
     """
     Checks if all tools are there.
     Exits with 0 if everything went okilydokily.
@@ -248,7 +247,7 @@ def main():
     The main program.
     """
     options = parse_arguments()
-    gitserver = preflight_checks(options)
+    gitserver = preflight_checks()
     if options['projects']:
         list_projects(gitserver, options)
     if options['issues']:

xml/notes/README.md

@@ -1,2 +1,2 @@
 # notes
-This folder holds all email correspondence and other notes
+This folder holds all email correspondence, notes and received source code

xml/source/futurework.xml

@@ -1,4 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <section id="futurework" xml:base="futurework.xml" break="before" inexecsummary="no">
   <title>Future Work</title>
+  <p>
+    <ul>
+      <li>
+        <b>Title</b><br/>
+        Description
+      </li>
+    </ul>
+  </p>
 </section>

xml/xslt/styles_inv.xslt

@@ -67,7 +67,7 @@
     <xsl:attribute-set name="logo">
         <xsl:attribute name="padding-top">0cm</xsl:attribute>
         <xsl:attribute name="padding-bottom">0cm</xsl:attribute>
-        <xsl:attribute name="src">url(../graphics/logo_alt.png)</xsl:attribute>
+        <xsl:attribute name="src">url(../graphics/logo.png)</xsl:attribute>
         <xsl:attribute name="width">30mm</xsl:attribute>
         <xsl:attribute name="content-width">scale-to-fit</xsl:attribute>
         <xsl:attribute name="content-height">scale-to-fit</xsl:attribute>
@@ -104,4 +104,4 @@
     <xsl:attribute-set name="table-shading">
         <xsl:attribute name="background-color">#EEEEEE</xsl:attribute>
     </xsl:attribute-set>
-</xsl:stylesheet>
+</xsl:stylesheet>