From 1f02bfe0eeccfb6ed158301332f941eb2783803e Mon Sep 17 00:00:00 2001 From: Peter Mosmans Date: Tue, 21 Feb 2017 19:35:18 +1100 Subject: [PATCH 1/2] Implemented markdown to XML conversion using pandoc --- chatops/python/gitlab-to-pentext.py | 89 ++++++++++++----------------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/chatops/python/gitlab-to-pentext.py b/chatops/python/gitlab-to-pentext.py index f047a57..d2ad9ed 100644 --- a/chatops/python/gitlab-to-pentext.py +++ b/chatops/python/gitlab-to-pentext.py @@ -24,28 +24,27 @@ from __future__ import print_function from __future__ import unicode_literals import argparse -import collections import os import sys import textwrap try: import gitlab - import jxmlease + import pypandoc # Path of this script. The validate_report module is on the same path. sys.path.append(os.path.dirname(__file__)) import validate_report except ImportError as exception: - print('[-] This script needs python-gitlab, jxmlease and validate_report library', + print('[-] This script needs python-gitlab, pypandoc and validate_report library', file=sys.stderr) print("validate_report is part of the pentext framework", file=sys.stderr) print("Install python-gitlab with: sudo pip install python-gitlab", file=sys.stderr) - print("Install jxmlease with: sudo pip install jxmlease", file=sys.stderr) - print("", file=sys.stderr) + print("Install pypandoc with: sudo pip install pypandoc\n", file=sys.stderr) print("Currently missing: " + exception.message, file=sys.stderr) sys.exit(-1) - +DECLARATION = '\n' + def add_finding(issue, options): """ Writes issue as XML finding to file. @@ -57,26 +56,21 @@ def add_finding(issue, options): finding_type = 'TODO' finding_id = '{0}-{1}'.format(issue.iid, valid_filename(title)) filename = 'findings/{0}.xml'.format(finding_id) - finding = collections.OrderedDict() - finding['title'] = title - finding['description'] = unicode.replace(issue.description, - '\r\n', '\n') - finding['technicaldescription'] = '' + finding = u'{0}\n'.format(title) + finding += '{0}\n\n'.format(convert_text(issue.description)) + technical_description = '' for note in [x for x in issue.notes.list() if not x.system]: - finding['technicaldescription'] += unicode.replace(note.body, - '\r\n', '\n') - finding['impact'] = {} - finding['impact']['p'] = 'TODO' - finding['recommendation'] = {} - finding['recommendation']['ul'] = {} - finding['recommendation']['ul']['li'] = 'TODO' - finding_xml = jxmlease.XMLDictNode(finding, tag='finding', - xml_attrs={'id': finding_id, - 'threatLevel': threat_level, - 'type': finding_type}) + technical_description += u'{0}\n'.format(convert_text(note.body)) + finding += '\n{0}\n\n'.format(technical_description) + finding += '\nTODO\n\n' + finding += '\n\n\n' + finding = u'{0}\n{4}\n'.format(DECLARATION, + finding_id, + threat_level, + finding_type, + finding) if options['dry_run']: - print_line('[+] {0}'.format(filename)) - print(finding_xml.emit_xml()) + print_line('[+] {0}\n{1}'.format(filename, finding)) else: if os.path.isfile(filename) and not options['overwrite']: print_line('Finding {0} already exists (use --overwrite to overwrite)'. @@ -84,8 +78,15 @@ def add_finding(issue, options): else: if options['y'] or ask_permission('Create file ' + filename): with open(filename, 'w') as xmlfile: - xmlfile.write(finding_xml.emit_xml().encode('utf-8')) - print_line('[+] Created {0}'.format(filename)) + xmlfile.write(finding) + print_line('[+] Created {0}'.format(filename)) + + +def convert_text(text): + """ + Convert (gitlab) markdown to 'XML' (actually HTML5). + """ + return unicode.replace(pypandoc.convert_text(text, 'html5', format='markdown_github'), '\r\n', '\n') def add_non_finding(issue, options): @@ -97,18 +98,15 @@ def add_non_finding(issue, options): title), options) non_finding_id = '{0}-{1}'.format(issue.iid, valid_filename(title)) filename = 'non-findings/{0}.xml'.format(non_finding_id) - non_finding = collections.OrderedDict() - non_finding['title'] = title - non_finding['p'] = unicode.replace(issue.description, - '\r\n', '\n') + non_finding = u'{0}\n{1}\n'.format(title, + convert_text(issue.description)) for note in [x for x in issue.notes.list() if not x.system]: - non_finding['p'] += unicode.replace(note.body, - '\r\n', '\n') - non_finding_xml = jxmlease.XMLDictNode(non_finding, tag='non-finding', - xml_attrs={'id': non_finding_id}) + non_finding += u'

{0}

\n'.format(convert_text(note.body)) + non_finding = u'{0}\n{2}\n\n'.format(DECLARATION, + non_finding_id, + non_finding) if options['dry_run']: - print_line('[+] {0}'.format(filename)) - print(non_finding_xml.emit_xml()) + print_line('[+] {0}\n{1}'.format(filename, non_finding)) else: if os.path.isfile(filename) and not options['overwrite']: print_line('Non-finding {0} already exists (use --overwrite to overwrite)'. @@ -116,8 +114,8 @@ def add_non_finding(issue, options): else: if options['y'] or ask_permission('Create file ' + filename): with open(filename, 'w') as xmlfile: - xmlfile.write(non_finding_xml.emit_xml().encode('utf-8')) - print_line('[+] Created {0}'.format(filename)) + xmlfile.write(non_finding) + print_line('[+] Created {0}'.format(filename)) def ask_permission(question): @@ -128,26 +126,13 @@ def ask_permission(question): return raw_input().lower() == 'y' -def convert_markdown(text): - """ - Replace markdown monospace with monospace tags - """ - result = text - return result - # print('EXAMINING ' + text + ' END') - # monospace = re.findall("\`\`\`(.*?)\`\`\`", text, re.DOTALL) - # if len(monospace): - # result = {} - # result['monospace'] = ''.join(monospace) - - def list_issues(gitserver, options): """ Lists all issues for options['issues'] """ try: for issue in gitserver.project_issues.list(project_id=options['issues'], - per_page=99): + per_page=999): if issue.state == 'closed' and not options['closed']: continue if 'finding' in [x.lower() for x in issue.labels]: From 04b9154e970f73e8af4a62fefa324e2bdaaaf088 Mon Sep 17 00:00:00 2001 From: Peter Mosmans Date: Tue, 21 Feb 2017 19:39:03 +1100 Subject: [PATCH 2/2] Added changed prerequisites for pandoc --- chatops/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/chatops/README.md b/chatops/README.md index 8edf7e7..ee933f9 100644 --- a/chatops/README.md +++ b/chatops/README.md @@ -37,10 +37,15 @@ The scripts use multiple environment variables, that can be set by the user unde ## Prerequisites -The Bash scripts use the python-gitlab command-line interface to talk to the gitlab instance. This interface can be installed using `sudo pip install git+https://github.com/gpocentek/python-gitlab`. Obviously, Python needs to be installed as well. +### python-gitlab +The Bash scripts use the *python-gitlab* command-line interface to talk to the gitlab instance. This interface can be installed using `sudo pip install git+https://github.com/gpocentek/python-gitlab`. Obviously, Python needs to be installed as well. This command line interface expects a configuration file `.python-gitlab.cfg` for the user under which rosbot is running, which it uses to connect to gitlab. Make sure it contains the correct details so that you can connect to gitlab. -If you want to convert and build documents, the pentext toolchain is necessary. Use the ansible playbook https://galaxy.ansible.com/PeterMosmans/docbuilder/ or install the tools (Java, Saxon and Apache FOP) by hand, see https://github.com/radicallyopensecurity/pentext/blob/master/xml/doc/Tools%20manual.md for more information. +If you want to convert and build documents, the pentext toolchain is necessary. Use the Ansible playbook https://galaxy.ansible.com/PeterMosmans/docbuilder/ or install the tools (Java, Saxon and Apache FOP) by hand, see https://github.com/radicallyopensecurity/pentext/blob/master/xml/doc/Tools%20manual.md for more information. + +### Python libraries +Pandoc is necessary in order to automatically convert gitlab issues written in markdown to XML format. +The *pypandoc* library is also necessary: `sudo pip install pypandoc`. ## Test the configuration Test out whether the configuration is successful by manually executing the Bash script [bash/test_pentext](bash/test_pentext) - this should return an OK.