diff --git a/chatops/bash/handler_build b/chatops/bash/handler_build index 500a00b..cd1177c 100644 --- a/chatops/bash/handler_build +++ b/chatops/bash/handler_build @@ -120,6 +120,7 @@ build() { add_to_repo() { git add $TARGETPDF git add target/waiver_?*.pdf &>/dev/null + git add target/execsummary.pdf &>/dev/null git commit -q -m "$TARGETPDF proudly manufactured using ChatOps" &>/dev/null git push -q >/dev/null } diff --git a/chatops/bash/releaser.sh b/chatops/bash/releaser.sh index e483420..0cdcdf6 100644 --- a/chatops/bash/releaser.sh +++ b/chatops/bash/releaser.sh @@ -42,7 +42,7 @@ if [ -f ${source} ]; then else cp -v ${source} ${fullname} PASS=$(head -c 25 /dev/random | base64 | head -c 25) - zip --password ${PASS} "${fullname}.zip" ${fullname} 2>/dev/null && echo "Zip file encrypted with password '${PASS}'" + 7z a -p${PASS} "${fullname}.zip" ${fullname} 2>/dev/null && echo "Zip file encrypted with password '${PASS}'" fi else echo "Could not find source ${source}" diff --git a/chatops/python/docbuilder.py b/chatops/python/docbuilder.py index 5da7c57..a1c85d6 100644 --- a/chatops/python/docbuilder.py +++ b/chatops/python/docbuilder.py @@ -32,6 +32,7 @@ import textwrap GITREV = 'GITREV' # Magic tag which gets replaced by the git short commit hash OFFERTE = 'generate_offerte.xsl' # XSL for generating waivers WAIVER = 'waiver_' # prefix for waivers +EXECSUMMARY = 'execsummary' # generating an executive summary instead of a report def parse_arguments(): @@ -55,6 +56,9 @@ the Free Software Foundation, either version 3 of the License, or help='overwrite output file if it already exists') parser.add_argument('-date', action='store', help='the invoice date') + parser.add_argument('-execsummary', action='store', + help="""create an executive summary as well as a report (true/false). + Default: false """) parser.add_argument('--fop-config', action='store', default='/etc/docbuilder/rosfop.xconf', help="""fop configuration file (default @@ -141,6 +145,8 @@ def to_fo(options): cmd.append('INVOICE_NO=' + options['invoice']) if options['date']: cmd.append('DATE=' + options['date']) + if options['execsummary']: + cmd.append('EXEC_SUMMARY=' + options['execsummary']) process = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() print_output(stdout, stderr) @@ -222,6 +228,23 @@ def main(): except OSError as exception: print_exit('[-] ERR: {0}'.format(exception.strerror), exception.errno) + if options['execsummary'] == 'true': # we're generating a summary as well as a report + report_output = options['output'] + verboseprint('generating additional executive summary') + output_dir = os.path.dirname(options['output']) + fop_dir = os.path.dirname(options['fop']) + try: + for fop in [os.path.splitext(x)[0] for x in + os.listdir(fop_dir) if x.endswith('fo')]: + if EXECSUMMARY in fop: + options['output'] = output_dir + os.sep + fop + '.pdf' + else: + options['output'] = report_output + options['fop'] = fop_dir + os.sep + fop + '.fo' + result = to_pdf(options) and result + except OSError as exception: + print_exit('[-] ERR: {0}'.format(exception.strerror), + exception.errno) else: result = to_pdf(options) diff --git a/chatops/python/gitlab-to-pentext.py b/chatops/python/gitlab-to-pentext.py index 6200c2a..9fd6e14 100644 --- a/chatops/python/gitlab-to-pentext.py +++ b/chatops/python/gitlab-to-pentext.py @@ -146,7 +146,8 @@ def list_issues(gitserver, options): Lists all issues for options['issues'] """ try: - for issue in gitserver.project_issues.list(project_id=options['issues']): + for issue in gitserver.project_issues.list(project_id=options['issues'], + per_page=99): if issue.state == 'closed' and not options['closed']: continue if 'finding' in issue.labels: @@ -251,7 +252,7 @@ def valid_filename(filename): """ result = '' for char in filename.strip(): - if char in [':', '/', '.', '\\', ' ', '[', ']', '(', ')', '\'']: + if char in ['*', ':', '/', '.', '\\', ' ', '[', ']', '(', ')', '\'']: if len(char) and not result.endswith('-'): result += '-' else: diff --git a/chatops/python/validate_report.py b/chatops/python/validate_report.py index fe9ae61..720acd5 100644 --- a/chatops/python/validate_report.py +++ b/chatops/python/validate_report.py @@ -23,6 +23,7 @@ from __future__ import print_function from __future__ import unicode_literals import argparse +import logging import mmap import os import re @@ -46,8 +47,10 @@ VOCABULARY = 'project-vocabulary.pws' # Snippets may contain XML fragments without the proper entities EXAMPLEDIR = 'examples/' NOT_CAPITALIZED = ['a', 'an', 'and', 'as', 'at', 'but', 'by', 'for', 'in', - 'nor', 'of', 'on', 'or', 'the', 'to', 'up'] + 'jQuery', 'jQuery-UI', 'nor', 'of', 'on', 'or', 'the', 'to', + 'up'] SNIPPETDIR = 'snippets/' +STATUS = 25 # loglevel for 'generic' status messages TEMPLATEDIR = 'templates/' OFFERTE = '/offerte.xml' REPORT = '/report.xml' @@ -60,8 +63,29 @@ if DOCBUILDER: import proxy_vagrant try: import aspell -except: - print('[-] aspell not installed: spelling not available') +except ImportError: + print('[-] aspell not installed: spelling not available',) + + +class LogFormatter(logging.Formatter): + """ + Format log messages according to their type. + """ + # DEBUG = (10) debug status messages + # INFO = (20) verbose status messages + # STATUS = (25) generic status messages + # WARNING = (30) warning messages (= errors in validation) + # ERROR = (40) error messages (= program errors) + FORMATS = {logging.DEBUG :"DEBUG: %(module)s: %(lineno)d: %(message)s", + logging.INFO : "[*] %(message)s", + STATUS : "[+] %(message)s", + logging.WARN : "[-] %(message)s", + logging.ERROR : "ERROR: %(message)s", + 'DEFAULT' : "%(message)s"} + + def format(self, record): + self._fmt = self.FORMATS.get(record.levelno, self.FORMATS['DEFAULT']) + return logging.Formatter.format(self, record) def parse_arguments(): @@ -106,21 +130,37 @@ the Free Software Foundation, either version 3 of the License, or return vars(parser.parse_args()) -def validate_spelling(tree, filename, options): +def initialize_speller(): """ - Checks spelling of text within tags. - If options['learn'], then unknown words will be added to the dictionary. + Initialize and return speller module. """ - result = True + speller = None try: speller = aspell.Speller(('lang', 'en'), ('personal-dir', '.'), ('personal', VOCABULARY)) - except: # some versions of aspell use a different path - speller = aspell.Speller(('lang', 'en'), - ('personal-path', './' + VOCABULARY)) - if options['debug']: - [print(i[0] + ' ' + str(i[2]) + '\n') for i in speller.ConfigKeys()] + except aspell.AspellConfigError as exception: # some versions of aspell use a different path + logging.debug('Encountered exception when trying to intialize spelling: %s', + exception) + try: + speller = aspell.Speller(('lang', 'en'), + ('personal-path', './' + VOCABULARY)) + except aspell.AspellSpellerError as exception: + logging.error('Could not initialize speller: %s', exception) + if speller: + [logging.debug('%s %s', i[0], i[2]) for i in speller.ConfigKeys()] + return speller + + +def validate_spelling(tree, filename, options, speller): + """ + Check spelling of text within tags. + If options['learn'], then unknown words will be added to the dictionary. + """ + result = True + if not speller: + options['spelling'] = False + return result try: root = tree.getroot() for section in root.iter(): @@ -132,12 +172,13 @@ def validate_spelling(tree, filename, options): speller.addtoPersonal(word) else: result = False - print('[-] Misspelled (unknown) word {0} in {1}'. - format(word.encode('utf-8'), filename)) + logging.warning('Misspelled (unknown) word %s in %s', + word.encode('utf-8'), filename) if options['learn']: speller.saveAllwords() except aspell.AspellSpellerError as exception: - print('[-] Spelling disabled ({0})'.format(exception)) + logging.error('Disabled spelling (%s)', exception) + options['spelling'] = False return result @@ -152,6 +193,9 @@ def all_files(): def open_editor(filename): + """ + Open editor with file to edit. + """ if sys.platform in ('linux', 'linux2'): editor = os.getenv('EDITOR') if editor: @@ -176,6 +220,7 @@ def validate_files(filenames, options): findings = [] non_findings = [] scans = [] + speller = initialize_speller() for filename in filenames: if (filename.lower().endswith('.xml') or filename.lower().endswith('xml"')): @@ -184,7 +229,7 @@ def validate_files(filenames, options): (REPORT in filename and not options['no_report']): masters.append(filename) # try: - type_result, xml_type = validate_xml(filename, options) + type_result, xml_type = validate_xml(filename, options, speller) result = result and type_result if 'non-finding' in xml_type: non_findings.append(filename) @@ -200,16 +245,6 @@ def validate_files(filenames, options): return result -def print_output(options, stdout, stderr=None): - """ - Prints out standard out and standard err using the verboseprint function. - """ - if stdout and options['verbose']: - print('[+] {0}'.format(stdout)) - if stderr and options['verbose']: - print('[-] {0}'.format(stderr)) - - def validate_report(): """ Validates XML report file by trying to build it. @@ -220,7 +255,7 @@ def validate_report(): return proxy_vagrant.execute_command(host, command) -def validate_xml(filename, options): +def validate_xml(filename, options, speller): """ Validates XML file by trying to parse it. Returns True if the file validated successfully. @@ -230,12 +265,12 @@ def validate_xml(filename, options): # crude check whether the file is outside the pentext framework if 'notes' in filename: return result, xml_type - print_output(options, 'Validating XML file: {0}'.format(filename)) + logging.info('Validating XML file: %s', filename) try: with open(filename, 'rb') as xml_file: xml.sax.parse(xml_file, xml.sax.ContentHandler()) tree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False)) - type_result, xml_type = validate_type(tree, filename, options) + type_result, xml_type = validate_type(tree, filename, options, speller) result = validate_long_lines(tree, filename, options) and result and type_result if options['edit'] and not result: open_editor(filename) @@ -282,7 +317,7 @@ def capitalize(line): return capitalized.strip() -def validate_type(tree, filename, options): +def validate_type(tree, filename, options, speller): """ Performs specific checks based on type. Currently only finding and non-finding are supported. @@ -294,7 +329,7 @@ def validate_type(tree, filename, options): attributes = [] tags = [] if options['spelling']: - result = validate_spelling(tree, filename, options) + result = validate_spelling(tree, filename, options, speller) if xml_type == 'pentest_report': attributes = ['findingCode'] if xml_type == 'finding': @@ -331,11 +366,11 @@ def validate_type(tree, filename, options): fix = True for tag in tags: if root.find(tag) is None: - print('[-] Missing tag in {0}: {1}'.format(filename, tag)) + logging.warning('Missing tag in %s: %s', filename, tag) result = False continue if not get_all_text(root.find(tag)): - print('[-] Empty tag in {0}: {1}'.format(filename, tag)) + logging.warning('Empty tag in %s: %s', filename, tag) result = False continue if tag == 'title' and (options['capitalization'] and \ @@ -404,14 +439,14 @@ def validate_master(filename, findings, non_findings, scans, options): result = True include_findings = [] include_nonfindings = [] - print_output(options, 'Validating master file {0}'.format(filename)) + logging.info('Validating master file %s', filename) try: xmltree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False)) if not find_keyword(xmltree, 'TODO', filename): print('[-] Keyword checks failed for {0}'.format(filename)) result = False - print_output(options, 'Performing cross check on findings, non-findings and scans...') + logging.info('Performing cross check on findings, non-findings and scans...') for finding in findings: if not cross_check_file(filename, finding): print('[A] Cross check failed for finding {0}'. @@ -420,23 +455,22 @@ def validate_master(filename, findings, non_findings, scans, options): result = False for non_finding in non_findings: if not cross_check_file(filename, non_finding): - print('[A] Cross check failed for non-finding {0}'. - format(non_finding)) + logging.warning('Cross check failed for non-finding %s', non_finding) include_nonfindings.append(non_finding) result = False if result: - print_output(options, 'Cross checks successful') + logging.info('Cross checks successful') except (ElementTree.ParseError, IOError) as exception: - print('[-] validating {0} failed ({1})'.format(filename, exception)) + logging.warning('Validating %s failed: %s', filename, exception) result = False if not result: if options['auto_fix']: add_include(filename, 'findings', include_findings) add_include(filename, 'nonFindings', include_nonfindings) close_file(filename) - print('[+] Automatically fixed {0}'.format(filename)) + logging.info('Automatically fixed %s', filename) else: - print('[+] NOTE: Items with [A] can be fixed automatically, use --auto-fix') + logging.warning('Item can be fixed automatically, use --auto-fix') return result @@ -448,7 +482,7 @@ def report_string(report_file): report = open(report_file) return mmap.mmap(report.fileno(), 0, access=mmap.ACCESS_READ) except IOError as exception: - print('[-] Could not open {0} ({1})'.format(report_file, exception)) + logging.critical('Could not open %s: %s', report_file, exception) sys.exit(-1) @@ -459,7 +493,7 @@ def cross_check_file(filename, external): result = True report_text = report_string(filename) if report_text.find(external) == -1: - print('[-] could not find a reference in {0} to {1}'.format(filename, external)) + logging.warning('Could not find a reference in %s to %s', filename, external) result = False return result @@ -507,11 +541,29 @@ def find_keyword(xmltree, keyword, filename): section = 'in {0}'.format(tag.attrib['id']) if tag.text: if keyword in tag.text: - print('[-] {0} found in {1} {2}'.format(keyword, filename, section)) + logging.warning('%s found in %s %s', keyword, filename, section) result = False return result +def setup_logging(options): + """ + Set up loghandlers according to options. + """ + logger = logging.getLogger() + logger.setLevel(0) + console = logging.StreamHandler(stream=sys.stdout) + console.setFormatter(LogFormatter()) + if options['debug']: + console.setLevel(logging.DEBUG) + else: + if options['verbose']: + console.setLevel(logging.INFO) + else: + logger.setLevel(STATUS) + logger.addHandler(console) + + def main(): """ The main program. Cross-checks, validates XML files and report. @@ -521,30 +573,24 @@ def main(): reload(sys) sys.setdefaultencoding('utf-8') options = parse_arguments() + setup_logging(options) if options['all']: options['capitalization'] = True options['long'] = True if options['learn']: - print_output(options, 'Adding unknown words to {0}'.format(VOCABULARY)) - # if options['spelling']: - # if not os.path.exists(VOCABULARY): - # print_output(options, 'Creating project-specific vocabulary file {0}'. - # format(VOCABULARY)) - # options['learn'] = True - print_output(options, 'Validating all XML files...') + logging.debug('Adding unknown words to %s', VOCABULARY) + logging.info('Validating all XML files...') result = validate_files(all_files(), options) if result: - print_output(options, 'Validation checks successful') if DOCBUILDER: - print_output(options, 'Validating report build...') + logging.info('Validating report build...') result = validate_report() and result if result: - print('[+] Succesfully validated everything. Good to go') + logging.log(STATUS, 'Validation checks successful. Good to go') else: - print('[-] Errors occurred') + logging.warning('Validation failed') if options['spelling'] and options['learn']: - print('[*] Don\'t forget to check the vocabulary file {0}'. - format(VOCABULARY)) + logging.log(STATUS('Don\'t forget to check the vocabulary file %s', VOCABULARY)) if __name__ == "__main__": diff --git a/scripts/gitlab-to-pentext.py b/scripts/gitlab-to-pentext.py index ce71e3d..a02fd6a 100644 --- a/scripts/gitlab-to-pentext.py +++ b/scripts/gitlab-to-pentext.py @@ -119,12 +119,11 @@ def convert_markdown(text): Replace markdown monospace with monospace tags """ result = text - return result + return result # currently not implemented print('EXAMINING ' + text + ' END') monospace = re.findall("\`\`\`(.*?)\`\`\`", text, re.DOTALL) print(monospace) if len(monospace): - print('YESSS ' + monospace) result = {} result['monospace'] = ''.join(monospace) @@ -158,7 +157,7 @@ def parse_arguments(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent('''\ -gitlab-to-pentext - imports and updates gitlab issues into PetText (XML) format +gitlab-to-pentext - imports and updates gitlab issues into PenText (XML) format Copyright (C) 2016 Peter Mosmans [Radically Open Security]] This program is free software: you can redistribute it and/or modify @@ -184,7 +183,7 @@ the Free Software Foundation, either version 3 of the License, or return vars(parser.parse_args()) -def preflight_checks(options): +def preflight_checks(): """ Checks if all tools are there. Exits with 0 if everything went okilydokily. @@ -248,7 +247,7 @@ def main(): The main program. """ options = parse_arguments() - gitserver = preflight_checks(options) + gitserver = preflight_checks() if options['projects']: list_projects(gitserver, options) if options['issues']: diff --git a/xml/notes/README.md b/xml/notes/README.md index ac04a43..9e4b845 100644 --- a/xml/notes/README.md +++ b/xml/notes/README.md @@ -1,2 +1,2 @@ # notes -This folder holds all email correspondence and other notes +This folder holds all email correspondence, notes and received source code diff --git a/xml/source/futurework.xml b/xml/source/futurework.xml index 41cbc6e..edfbd91 100644 --- a/xml/source/futurework.xml +++ b/xml/source/futurework.xml @@ -1,4 +1,12 @@
Future Work +

+

+

diff --git a/xml/templates/pentestreport/report.xml b/xml/templates/pentestreport/report.xml index 86b9669..f498b32 100644 --- a/xml/templates/pentestreport/report.xml +++ b/xml/templates/pentestreport/report.xml @@ -81,6 +81,19 @@

Through automated scans we were able to gain the following information about the software and infrastructure. Detailed scan output can be found in the sections below.

+ + + + + +
Fingerprinted Information
+ +
Automated Scans diff --git a/xml/xslt/styles_inv.xslt b/xml/xslt/styles_inv.xslt index 31e4c06..cf66bfc 100644 --- a/xml/xslt/styles_inv.xslt +++ b/xml/xslt/styles_inv.xslt @@ -67,7 +67,7 @@ 0cm 0cm - url(../graphics/logo_alt.png) + url(../graphics/logo.png) 30mm scale-to-fit scale-to-fit @@ -104,4 +104,4 @@ #EEEEEE - \ No newline at end of file +