A Converter for McAfee SIEM Rule Export Files to Markdown Documents
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

383 lines
13 KiB

#!/usr/bin/env python3
'''
esm2markdown - McAfee ESM correlation rule XML export to markdown converter
Copyright (C) 2018 Michael Clemens
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
'''
import sys
import os.path
import re
from configparser import ConfigParser
from lxml import etree
from urllib.parse import unquote
from subprocess import check_call
# Set configuration defaults in case of missing ini file
key_style = "**"
value_style = ""
sort_rules = True
toc = True
imagepath = "images"
mklines = []
# Reads ini file, sets global variables
def readConfig():
# Overwrite variables with settings from ini file
try:
config = ConfigParser()
config.read('esm2markdown.ini')
key_style = config.get('config', 'key_style')
value_style = config.get('config', 'value_style')
sort_rules = config.getboolean('config', 'sort_rules')
toc = config.getboolean('config', 'toc')
imagepath = config.get('config', 'imagepath')
except:
print("Configuration file not found, using default settings")
# Generates a line containing linebreaks, indented lists, styles etc.
def addLine(typ,level,key,value):
lvl = ""
output = ""
valout = ""
if value:
value = unquote(value)
if key:
if typ == "list":
if level == 1: lvl = "* "
elif level == 2: lvl = " * "
elif level == 3: lvl = " * "
if value == "N/A": output = lvl + key_style + key + key_style + "\n"
elif value: output = lvl + key_style + key + key_style + " " + \
value_style + value + value_style + "\n"
elif typ == "header":
if level == 1: lvl = "# "
elif level == 2: lvl = "## "
elif level == 3: lvl = "### "
elif level == 4: lvl = "#### "
output = "\n" + lvl + key + "\n"
elif typ == "none":
output = key + "\n"
output = re.sub('\$\$',"!",output)
mklines.append(output)
def validateXML(xmlfile):
xmlok = True
parser = etree.XMLParser(strip_cdata=False)
with open(xmlfile, "rb") as source:
root = etree.parse(source, parser=parser)
temp = root.find("rules")
msglist = []
for e in temp:
msg = e.findtext("message")
if msg in msglist:
print("Duplicate rule: " + msg)
xmlok = False
msglist.append(msg)
if xmlok == False:
print("The XML file contains some errors. Please fix the input file and rerun this script.")
return xmlok
# Sorts input XML alphabetically based on Rule Names
def sortXML(xmlfile):
parser = etree.XMLParser(strip_cdata=False)
with open(xmlfile, "rb") as source:
root = etree.parse(source, parser=parser)
temp = root.find("rules")
data = []
for e in temp:
msg = e.findtext("message")
data.append((msg, e))
data.sort()
temp[:] = [item[-1] for item in data]
return root
# Generates Markdown Syntax for Images
def addImage(rulename):
out = ""
if not os.path.exists(imagepath):
os.makedirs(imagepath)
imagefile = imagepath + "/" + rulename + ".png"
imagefile = imagefile.replace(" ", "_")
out = "![](" + imagefile + ")\n\n\n"
return out
# Generates dict object with relations between triggers and match blocks
def getRelationDict(cdata):
rel = {}
# Populate Tree with rule objects
for r in cdata.getiterator('rule'):
if not r.get('name') == "Root Rule":
for e in r.iter():
if str(e.tag) == 'action':
if e.get('type') == "TRIGGER":
for trigger in cdata.getiterator('trigger'):
if e.get('trigger') == trigger.get('name'):
rel[r.get('name')]=trigger.get('name')
# Populate Tree with trigger objects
for t in cdata.getiterator('trigger'):
tname = t.get('name')
tparent = t.findtext('trigger')
if tname:
if not tparent:
tparent = "root"
rel[tname]=tparent
return rel
# Populates Graph object with trigger nodes and edges
def addTriggersToGraph(cdata,G):
# get dictionary with all element relations
reldict = getRelationDict(cdata)
# Walk through dict
tco = 99
for key in sorted(reldict):
trigcount = 0
# count triggers per rule
if key.startswith("trigger") or key.startswith("Root Trigger"):
for trigkey in sorted(reldict):
if reldict[trigkey] == key:
trigcount += 1
# get count value from triggers element
for tc in cdata.getiterator('trigger'):
if tc.get('name') == key:
if tc.get('count'):
tco = int(tc.get('count'))
# compare count value and counted triggers
# if both are the same, all match blocks need to match
# therefore the logical operator is AND
# otherwise its OR
if tco == trigcount:
oper = "AND"
else:
oper = "OR"
# add trigger nodes to graph
G.add_node(key, label=oper, shape='plaintext')
if key != "root" and reldict[key] != "root":
G.add_edge(reldict[key],key,splines='ortho', nodesep=0.2)
return G
# Generates png file visualizing match block relationships
def generateGraph(cdata,dMatchBlocks,rid):
dependencies = True
try:
from networkx.drawing.nx_pydot import write_dot
import networkx as nx
except ImportError:
print("Cannot find networkx. Please install pydot and networkx.")
print("Output file will be created without images.")
dependencies = False
return False
if dependencies:
G = nx.DiGraph()
# populate Graph object with triggers
G = addTriggersToGraph(cdata,G)
for x in dMatchBlocks:
G.add_node(x, color=dMatchBlocks[x]['shapecol'],style='filled',fillcolor=dMatchBlocks[x]['shapecol'],shape='box')
G.add_edge(dMatchBlocks[x]['parent'],x)
# write dot file for Graphviz out to file system
write_dot(G,'file.dot')
# execute 'dot' as os command, generate png file from dot file
try:
check_call(['dot','-Tpng','-Grankdir=LR','file.dot','-o',imagepath + '/' +
rid +'.png'])
except OSError as e:
dependencies = False
print("'dot' could not be found. Please install pydot.")
return True
# Walks through list 'mklines' and write content to outfile
def writeMarkdownFile(outfile):
file = open(outfile,"w")
for l in mklines:
file.write(l)
file.close()
# Parses and procresses XML file
def parseXML(xmlfile):
mklines.clear
if sort_rules:
root = sortXML(xmlfile)
else:
root = etree.parse(xmlfile)
if toc:
addLine("header",1,"Correlation Rule Overview","")
for rule in root.getiterator('rule'):
addLine("list",1,rule.findtext('message'),"N/A")
for rule in root.getiterator('rule'):
dMatchBlocks = {}
# Get CDATA
text = rule.findtext('text')
cdata = etree.fromstring(text)
# Print rule name as header
addLine("header",1,rule.findtext('message'),"")
# Print rule description
addLine("header",2,"Description","")
addLine("none","",rule.findtext('description'),"")
# Print rule information (ID, Normalization, Severity, Tags, Group By)
addLine("header",2,"General Information","")
addLine("list",1,"Rule ID:",rule.findtext('id'))
addLine("list",1,"Normalization ID:",rule.findtext('normid'))
addLine("list",1,"Severity:",rule.findtext('severity'))
for tags in rule.getiterator('tag'):
addLine("list",1,"Tag:",tags.text)
for rs in cdata.getiterator('ruleset'):
addLine("list",1,"Group By:",rs.get('correlationField'))
addLine("header",2,"Correlation Details","")
# Insert diagram into text
mklines.append(addImage(rule.findtext('id')))
parameters = False
# Print rule parameters
for param in cdata.getiterator('param'):
if not parameters:
addLine("header",3,"Parameters","")
parameters = True
addLine("list",1,param.get('name'),"N/A")
addLine("list",2,"Description:",param.get('description'))
addLine("list",2,"Default Value:",param.get('defaultvalue'))
addLine("header",3,"Rules","")
# Parse CDATA element and print correlation rule match blocks
for r in cdata.getiterator('rule'):
# initialize variables
o = ""
v = ""
t = ""
override = ""
parent = ""
matchtype = ""
# Walk through all rules except Root Rule
if not r.get('name') == "Root Rule":
addLine("header",4,r.get('name').title().replace("_", " "),"")
override = r.get('correlationField')
for e in r.iter():
if str(e.tag) == 'activate':
addLine("list",1,"Activate:",e.get('type'))
if override:
addLine("list",1,"Override Group By:",override)
if str(e.tag) == 'action':
if e.get('type') == "TRIGGER":
addLine("list",1,"Action:","Trigger")
# Find parent trigger of current rule
for trigger in cdata.getiterator('trigger'):
if e.get('trigger') == trigger.get('name'):
parent = trigger.get('name')
addLine("list",2,"Timeout:",trigger.get('timeout'))
addLine("list",2,"Time Units:",trigger.get('timeUnit'))
addLine("list",2,"Threshold:",trigger.get('threshold'))
addLine("list",2,"Sequence:",trigger.get('ordered'))
else:
addLine("list",1,"Action","N/A")
addLine("list",2,"NOT IMPLEMENTED","N/A")
if str(e.tag) == 'match':
matchtype = e.get('matchType')
addLine("list",1,"Match Type:",matchtype)
addLine("list",2,"Count:",e.get('count'))
if str(e.tag) == 'matchFilter':
addLine("list",1,"Match Filter","N/A")
if str(e.tag) == 'singleFilterComponent':
t = e.get('type')
if str(e.tag) == 'filterData':
if (e.get('name') == "operator"):
o = e.get('value')
if (e.get('name') == "value"):
v = e.get('value')
if o and v and t:
addLine("list",2,"Filter Component","N/A")
addLine("list",3,"Condition:","'" + t + "' " + o + " '" \
+ v + "'")
# Set nice label, add rule as graphviz node,
# add edge between trigger and node
label = t + r"\n" + o + r"\n" + v
v = ""
o = ""
if matchtype == "REFERENCE":
shapecol="blue"
else:
shapecol="orange"
if parent and r.get('name'):
nicename = r.get('name').title().replace("_", " ")
dMatchBlocks[nicename] = {'parent': parent, 'shapecol': shapecol}
mklines.append("\n\\newpage\n")
generateGraph(cdata,dMatchBlocks,rule.findtext('id'))
# Main function
if __name__=="__main__":
if len(sys.argv) != 3:
print('Invalid Numbers of Arguments. Script will be terminated.')
print('Usage: python esm2markdown <rule xml file> <output file>')
print('Example: python esm2markdown RuleExport.xml documentation.mk')
else:
readConfig()
if validateXML(sys.argv[1]) == True:
parseXML(sys.argv[1])
writeMarkdownFile(sys.argv[2])