DeforaNetworks/pentext
DeforaNetworks/pentext
3
0
Fork 0
Code Releases Activity

Merge branch 'master' of https://github.com/radicallyopensecurity/pentext into development

Browse Source 
# Conflicts:
#	chatops/bash/handler_build
This commit is contained in:
skyanth 2016-11-30 16:31:50 +01:00
commit dcb753f0e1
9 changed files with 146 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
chatops/bash/handler_build
View File

@ -120,6 +120,7 @@ build() {
add_to_repo() {
git add $TARGETPDF
git add target/waiver_?*.pdf &>/dev/null
git add target/execsummary.pdf &>/dev/null
git commit -q -m "$TARGETPDF proudly manufactured using ChatOps" &>/dev/null
git push -q >/dev/null
}

2
chatops/bash/releaser.sh
View File

@ -42,7 +42,7 @@ if [ -f ${source} ]; then
else
cp -v ${source} ${fullname}
PASS=$(head -c 25 /dev/random | base64 | head -c 25)
zip --password ${PASS} "${fullname}.zip" ${fullname} 2>/dev/null && echo "Zip file encrypted with password '${PASS}'"
7z a -p${PASS} "${fullname}.zip" ${fullname} 2>/dev/null && echo "Zip file encrypted with password '${PASS}'"
fi
else
echo "Could not find source ${source}"

23
chatops/python/docbuilder.py
View File

@ -32,6 +32,7 @@ import textwrap
GITREV = 'GITREV' # Magic tag which gets replaced by the git short commit hash
OFFERTE = 'generate_offerte.xsl' # XSL for generating waivers
WAIVER = 'waiver_' # prefix for waivers
EXECSUMMARY = 'execsummary' # generating an executive summary instead of a report
def parse_arguments():
@ -55,6 +56,9 @@ the Free Software Foundation, either version 3 of the License, or
help='overwrite output file if it already exists')
parser.add_argument('-date', action='store',
help='the invoice date')
parser.add_argument('-execsummary', action='store',
help="""create an executive summary as well as a report (true/false).
Default: false """)
parser.add_argument('--fop-config', action='store',
default='/etc/docbuilder/rosfop.xconf',
help="""fop configuration file (default
@ -141,6 +145,8 @@ def to_fo(options):
cmd.append('INVOICE_NO=' + options['invoice'])
if options['date']:
cmd.append('DATE=' + options['date'])
if options['execsummary']:
cmd.append('EXEC_SUMMARY=' + options['execsummary'])
process = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate()
print_output(stdout, stderr)
@ -222,6 +228,23 @@ def main():
except OSError as exception:
print_exit('[-] ERR: {0}'.format(exception.strerror),
exception.errno)
if options['execsummary'] == 'true': # we're generating a summary as well as a report
report_output = options['output']
verboseprint('generating additional executive summary')
output_dir = os.path.dirname(options['output'])
fop_dir = os.path.dirname(options['fop'])
try:
for fop in [os.path.splitext(x)[0] for x in
os.listdir(fop_dir) if x.endswith('fo')]:
if EXECSUMMARY in fop:
options['output'] = output_dir + os.sep + fop + '.pdf'
else:
options['output'] = report_output
options['fop'] = fop_dir + os.sep + fop + '.fo'
result = to_pdf(options) and result
except OSError as exception:
print_exit('[-] ERR: {0}'.format(exception.strerror),
exception.errno)
else:
result = to_pdf(options)

5
chatops/python/gitlab-to-pentext.py
View File

@ -146,7 +146,8 @@ def list_issues(gitserver, options):
Lists all issues for options['issues']
"""
try:
for issue in gitserver.project_issues.list(project_id=options['issues']):
for issue in gitserver.project_issues.list(project_id=options['issues'],
per_page=99):
if issue.state == 'closed' and not options['closed']:
continue
if 'finding' in issue.labels:
@ -251,7 +252,7 @@ def valid_filename(filename):
"""
result = ''
for char in filename.strip():
if char in [':', '/', '.', '\\', ' ', '[', ']', '(', ')', '\'']:
if char in ['*', ':', '/', '.', '\\', ' ', '[', ']', '(', ')', '\'']:
if len(char) and not result.endswith('-'):
result += '-'
else:

160
chatops/python/validate_report.py
View File

@ -23,6 +23,7 @@ from __future__ import print_function
from __future__ import unicode_literals
import argparse
import logging
import mmap
import os
import re
@ -46,8 +47,10 @@ VOCABULARY = 'project-vocabulary.pws'
# Snippets may contain XML fragments without the proper entities
EXAMPLEDIR = 'examples/'
NOT_CAPITALIZED = ['a', 'an', 'and', 'as', 'at', 'but', 'by', 'for', 'in',
'nor', 'of', 'on', 'or', 'the', 'to', 'up']
'jQuery', 'jQuery-UI', 'nor', 'of', 'on', 'or', 'the', 'to',
'up']
SNIPPETDIR = 'snippets/'
STATUS = 25 # loglevel for 'generic' status messages
TEMPLATEDIR = 'templates/'
OFFERTE = '/offerte.xml'
REPORT = '/report.xml'
@ -60,8 +63,29 @@ if DOCBUILDER:
import proxy_vagrant
try:
import aspell
except:
print('[-] aspell not installed: spelling not available')
except ImportError:
print('[-] aspell not installed: spelling not available',)
class LogFormatter(logging.Formatter):
"""
Format log messages according to their type.
"""
# DEBUG = (10) debug status messages
# INFO = (20) verbose status messages
# STATUS = (25) generic status messages
# WARNING = (30) warning messages (= errors in validation)
# ERROR = (40) error messages (= program errors)
FORMATS = {logging.DEBUG :"DEBUG: %(module)s: %(lineno)d: %(message)s",
logging.INFO : "[*] %(message)s",
STATUS : "[+] %(message)s",
logging.WARN : "[-] %(message)s",
logging.ERROR : "ERROR: %(message)s",
'DEFAULT' : "%(message)s"}
def format(self, record):
self._fmt = self.FORMATS.get(record.levelno, self.FORMATS['DEFAULT'])
return logging.Formatter.format(self, record)
def parse_arguments():
@ -106,21 +130,37 @@ the Free Software Foundation, either version 3 of the License, or
return vars(parser.parse_args())
def validate_spelling(tree, filename, options):
def initialize_speller():
"""
Checks spelling of text within tags.
If options['learn'], then unknown words will be added to the dictionary.
Initialize and return speller module.
"""
result = True
speller = None
try:
speller = aspell.Speller(('lang', 'en'),
('personal-dir', '.'),
('personal', VOCABULARY))
except: # some versions of aspell use a different path
speller = aspell.Speller(('lang', 'en'),
('personal-path', './' + VOCABULARY))
if options['debug']:
[print(i[0] + ' ' + str(i[2]) + '\n') for i in speller.ConfigKeys()]
except aspell.AspellConfigError as exception: # some versions of aspell use a different path
logging.debug('Encountered exception when trying to intialize spelling: %s',
exception)
try:
speller = aspell.Speller(('lang', 'en'),
('personal-path', './' + VOCABULARY))
except aspell.AspellSpellerError as exception:
logging.error('Could not initialize speller: %s', exception)
if speller:
[logging.debug('%s %s', i[0], i[2]) for i in speller.ConfigKeys()]
return speller
def validate_spelling(tree, filename, options, speller):
"""
Check spelling of text within tags.
If options['learn'], then unknown words will be added to the dictionary.
"""
result = True
if not speller:
options['spelling'] = False
return result
try:
root = tree.getroot()
for section in root.iter():
@ -132,12 +172,13 @@ def validate_spelling(tree, filename, options):
speller.addtoPersonal(word)
else:
result = False
print('[-] Misspelled (unknown) word {0} in {1}'.
format(word.encode('utf-8'), filename))
logging.warning('Misspelled (unknown) word %s in %s',
word.encode('utf-8'), filename)
if options['learn']:
speller.saveAllwords()
except aspell.AspellSpellerError as exception:
print('[-] Spelling disabled ({0})'.format(exception))
logging.error('Disabled spelling (%s)', exception)
options['spelling'] = False
return result
@ -152,6 +193,9 @@ def all_files():
def open_editor(filename):
"""
Open editor with file to edit.
"""
if sys.platform in ('linux', 'linux2'):
editor = os.getenv('EDITOR')
if editor:
@ -176,6 +220,7 @@ def validate_files(filenames, options):
findings = []
non_findings = []
scans = []
speller = initialize_speller()
for filename in filenames:
if (filename.lower().endswith('.xml') or
filename.lower().endswith('xml"')):
@ -184,7 +229,7 @@ def validate_files(filenames, options):
(REPORT in filename and not options['no_report']):
masters.append(filename)
# try:
type_result, xml_type = validate_xml(filename, options)
type_result, xml_type = validate_xml(filename, options, speller)
result = result and type_result
if 'non-finding' in xml_type:
non_findings.append(filename)
@ -200,16 +245,6 @@ def validate_files(filenames, options):
return result
def print_output(options, stdout, stderr=None):
"""
Prints out standard out and standard err using the verboseprint function.
"""
if stdout and options['verbose']:
print('[+] {0}'.format(stdout))
if stderr and options['verbose']:
print('[-] {0}'.format(stderr))
def validate_report():
"""
Validates XML report file by trying to build it.
@ -220,7 +255,7 @@ def validate_report():
return proxy_vagrant.execute_command(host, command)
def validate_xml(filename, options):
def validate_xml(filename, options, speller):
"""
Validates XML file by trying to parse it.
Returns True if the file validated successfully.
@ -230,12 +265,12 @@ def validate_xml(filename, options):
# crude check whether the file is outside the pentext framework
if 'notes' in filename:
return result, xml_type
print_output(options, 'Validating XML file: {0}'.format(filename))
logging.info('Validating XML file: %s', filename)
try:
with open(filename, 'rb') as xml_file:
xml.sax.parse(xml_file, xml.sax.ContentHandler())
tree = ElementTree.parse(filename, ElementTree.XMLParser(strip_cdata=False))
type_result, xml_type = validate_type(tree, filename, options)
type_result, xml_type = validate_type(tree, filename, options, speller)
result = validate_long_lines(tree, filename, options) and result and type_result
if options['edit'] and not result:
open_editor(filename)
@ -282,7 +317,7 @@ def capitalize(line):
return capitalized.strip()
def validate_type(tree, filename, options):
def validate_type(tree, filename, options, speller):
"""
Performs specific checks based on type.
Currently only finding and non-finding are supported.
@ -294,7 +329,7 @@ def validate_type(tree, filename, options):
attributes = []
tags = []
if options['spelling']:
result = validate_spelling(tree, filename, options)
result = validate_spelling(tree, filename, options, speller)
if xml_type == 'pentest_report':
attributes = ['findingCode']
if xml_type == 'finding':
@ -331,11 +366,11 @@ def validate_type(tree, filename, options):
fix = True
for tag in tags:
if root.find(tag) is None:
print('[-] Missing tag in {0}: {1}'.format(filename, tag))
logging.warning('Missing tag in %s: %s', filename, tag)
result = False
continue
if not get_all_text(root.find(tag)):
print('[-] Empty tag in {0}: {1}'.format(filename, tag))
logging.warning('Empty tag in %s: %s', filename, tag)
result = False
continue
if tag == 'title' and (options['capitalization'] and \
@ -404,14 +439,14 @@ def validate_master(filename, findings, non_findings, scans, options):
result = True
include_findings = []
include_nonfindings = []
print_output(options, 'Validating master file {0}'.format(filename))
logging.info('Validating master file %s', filename)
try:
xmltree = ElementTree.parse(filename,
ElementTree.XMLParser(strip_cdata=False))
if not find_keyword(xmltree, 'TODO', filename):
print('[-] Keyword checks failed for {0}'.format(filename))
result = False
print_output(options, 'Performing cross check on findings, non-findings and scans...')
logging.info('Performing cross check on findings, non-findings and scans...')
for finding in findings:
if not cross_check_file(filename, finding):
print('[A] Cross check failed for finding {0}'.
@ -420,23 +455,22 @@ def validate_master(filename, findings, non_findings, scans, options):
result = False
for non_finding in non_findings:
if not cross_check_file(filename, non_finding):
print('[A] Cross check failed for non-finding {0}'.
format(non_finding))
logging.warning('Cross check failed for non-finding %s', non_finding)
include_nonfindings.append(non_finding)
result = False
if result:
print_output(options, 'Cross checks successful')
logging.info('Cross checks successful')
except (ElementTree.ParseError, IOError) as exception:
print('[-] validating {0} failed ({1})'.format(filename, exception))
logging.warning('Validating %s failed: %s', filename, exception)
result = False
if not result:
if options['auto_fix']:
add_include(filename, 'findings', include_findings)
add_include(filename, 'nonFindings', include_nonfindings)
close_file(filename)
print('[+] Automatically fixed {0}'.format(filename))
logging.info('Automatically fixed %s', filename)
else:
print('[+] NOTE: Items with [A] can be fixed automatically, use --auto-fix')
logging.warning('Item can be fixed automatically, use --auto-fix')
return result
@ -448,7 +482,7 @@ def report_string(report_file):
report = open(report_file)
return mmap.mmap(report.fileno(), 0, access=mmap.ACCESS_READ)
except IOError as exception:
print('[-] Could not open {0} ({1})'.format(report_file, exception))
logging.critical('Could not open %s: %s', report_file, exception)
sys.exit(-1)
@ -459,7 +493,7 @@ def cross_check_file(filename, external):
result = True
report_text = report_string(filename)
if report_text.find(external) == -1:
print('[-] could not find a reference in {0} to {1}'.format(filename, external))
logging.warning('Could not find a reference in %s to %s', filename, external)
result = False
return result
@ -507,11 +541,29 @@ def find_keyword(xmltree, keyword, filename):
section = 'in {0}'.format(tag.attrib['id'])
if tag.text:
if keyword in tag.text:
print('[-] {0} found in {1} {2}'.format(keyword, filename, section))
logging.warning('%s found in %s %s', keyword, filename, section)
result = False
return result
def setup_logging(options):
"""
Set up loghandlers according to options.
"""
logger = logging.getLogger()
logger.setLevel(0)
console = logging.StreamHandler(stream=sys.stdout)
console.setFormatter(LogFormatter())
if options['debug']:
console.setLevel(logging.DEBUG)
else:
if options['verbose']:
console.setLevel(logging.INFO)
else:
logger.setLevel(STATUS)
logger.addHandler(console)
def main():
"""
The main program. Cross-checks, validates XML files and report.
@ -521,30 +573,24 @@ def main():
reload(sys)
sys.setdefaultencoding('utf-8')
options = parse_arguments()
setup_logging(options)
if options['all']:
options['capitalization'] = True
options['long'] = True
if options['learn']:
print_output(options, 'Adding unknown words to {0}'.format(VOCABULARY))
# if options['spelling']:
# if not os.path.exists(VOCABULARY):
# print_output(options, 'Creating project-specific vocabulary file {0}'.
# format(VOCABULARY))
# options['learn'] = True
print_output(options, 'Validating all XML files...')
logging.debug('Adding unknown words to %s', VOCABULARY)
logging.info('Validating all XML files...')
result = validate_files(all_files(), options)
if result:
print_output(options, 'Validation checks successful')
if DOCBUILDER:
print_output(options, 'Validating report build...')
logging.info('Validating report build...')
result = validate_report() and result
if result:
print('[+] Succesfully validated everything. Good to go')
logging.log(STATUS, 'Validation checks successful. Good to go')
else:
print('[-] Errors occurred')
logging.warning('Validation failed')
if options['spelling'] and options['learn']:
print('[*] Don\'t forget to check the vocabulary file {0}'.
format(VOCABULARY))
logging.log(STATUS('Don\'t forget to check the vocabulary file %s', VOCABULARY))
if __name__ == "__main__":

9
scripts/gitlab-to-pentext.py
View File

@ -119,12 +119,11 @@ def convert_markdown(text):
Replace markdown monospace with monospace tags
"""
result = text
return result
return result # currently not implemented
print('EXAMINING ' + text + ' END')
monospace = re.findall("\`\`\`(.*?)\`\`\`", text, re.DOTALL)
print(monospace)
if len(monospace):
print('YESSS ' + monospace)
result = {}
result['monospace'] = ''.join(monospace)
@ -158,7 +157,7 @@ def parse_arguments():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
gitlab-to-pentext - imports and updates gitlab issues into PetText (XML) format
gitlab-to-pentext - imports and updates gitlab issues into PenText (XML) format
Copyright (C) 2016 Peter Mosmans [Radically Open Security]]
This program is free software: you can redistribute it and/or modify
@ -184,7 +183,7 @@ the Free Software Foundation, either version 3 of the License, or
return vars(parser.parse_args())
def preflight_checks(options):
def preflight_checks():
"""
Checks if all tools are there.
Exits with 0 if everything went okilydokily.
@ -248,7 +247,7 @@ def main():
The main program.
"""
options = parse_arguments()
gitserver = preflight_checks(options)
gitserver = preflight_checks()
if options['projects']:
list_projects(gitserver, options)
if options['issues']:

2
xml/notes/README.md
View File

@ -1,2 +1,2 @@
# notes
This folder holds all email correspondence and other notes
This folder holds all email correspondence, notes and received source code

8
xml/source/futurework.xml
View File

@ -1,4 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<section id="futurework" xml:base="futurework.xml" break="before" inexecsummary="no">
<title>Future Work</title>
<p>
<ul>
<li>
<b>Title</b><br/>
Description
</li>
</ul>
</p>
</section>

4
xml/xslt/styles_inv.xslt
View File

@ -67,7 +67,7 @@
<xsl:attribute-set name="logo">
<xsl:attribute name="padding-top">0cm</xsl:attribute>
<xsl:attribute name="padding-bottom">0cm</xsl:attribute>
<xsl:attribute name="src">url(../graphics/logo_alt.png)</xsl:attribute>
<xsl:attribute name="src">url(../graphics/logo.png)</xsl:attribute>
<xsl:attribute name="width">30mm</xsl:attribute>
<xsl:attribute name="content-width">scale-to-fit</xsl:attribute>
<xsl:attribute name="content-height">scale-to-fit</xsl:attribute>
@ -104,4 +104,4 @@
<xsl:attribute-set name="table-shading">
<xsl:attribute name="background-color">#EEEEEE</xsl:attribute>
</xsl:attribute-set>
</xsl:stylesheet>
</xsl:stylesheet>