Skip to content

Commit b537cf9

Browse files
committed
Merge remote-tracking branch 'base/master'
2 parents 58704e9 + 8e4c688 commit b537cf9

16 files changed

+665
-356
lines changed

Pipfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pypdns = "*"
1818
pypssl = "*"
1919
pyeupi = "*"
2020
uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"}
21-
pymisp = {editable = true,extras = ["fileobjects,openioc,virustotal,pdfexport"],git = "https://github.com/MISP/PyMISP.git"}
21+
pymisp = {editable = true,extras = ["fileobjects,openioc,pdfexport"],git = "https://github.com/MISP/PyMISP.git"}
2222
pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"}
2323
pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"}
2424
pytesseract = "*"
@@ -60,6 +60,7 @@ geoip2 = "*"
6060
apiosintDS = "*"
6161
assemblyline_client = "*"
6262
vt-graph-api = "*"
63+
trustar = "*"
6364

6465
[requires]
6566
python_version = "3"

Pipfile.lock

+398-344
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

README.md

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
5757
* [Lastline query](misp_modules/modules/expansion/lastline_query.py) - Query Lastline with the link to an analysis and parse the report.
5858
* [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
5959
* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
60+
* [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload.
6061
* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
6162
* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
6263
* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).

REQUIREMENTS

+1
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ sparqlwrapper==1.8.5
9595
stix2-patterns==1.3.0
9696
tabulate==0.8.7
9797
tornado==6.0.4
98+
trustar==0.3.28
9899
url-normalize==1.4.1
99100
urlarchiver==0.2
100101
urllib3==1.25.8

doc/README.md

+45
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,22 @@ Module to access Macvendors API.
715715
716716
-----
717717

718+
#### [malwarebazaar](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/malwarebazaar.py)
719+
720+
Query the MALWAREbazaar API to get additional information about the input hash attribute.
721+
- **features**:
722+
>The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.
723+
>
724+
>The module is using the new format of modules able to return object since the result is one or multiple MISP object(s).
725+
- **input**:
726+
>A hash attribute (md5, sha1 or sha256).
727+
- **output**:
728+
>File object(s) related to the input attribute found on MALWAREbazaar databases.
729+
- **references**:
730+
>https://bazaar.abuse.ch/
731+
732+
-----
733+
718734
#### [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py)
719735

720736
Module to process some optical character recognition on pictures.
@@ -1168,6 +1184,35 @@ Module to get information from ThreatMiner.
11681184
11691185
-----
11701186

1187+
#### [trustar_enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/trustar_enrich.py)
1188+
1189+
<img src=logos/trustar.png height=60>
1190+
1191+
Module to get enrich indicators with TruSTAR.
1192+
- **features**:
1193+
>This module enriches MISP attributes with scoring and metadata from TruSTAR.
1194+
>
1195+
>The TruSTAR indicator summary is appended to the attributes along with links to any associated reports.
1196+
- **input**:
1197+
>Any of the following MISP attributes:
1198+
>- btc
1199+
>- domain
1200+
>- email-src
1201+
>- filename
1202+
>- hostname
1203+
>- ip-src
1204+
>- ip-dst
1205+
>- md5
1206+
>- sha1
1207+
>- sha256
1208+
>- url
1209+
- **output**:
1210+
>MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level score and additional source and scoring info.
1211+
- **references**:
1212+
>https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html
1213+
1214+
-----
1215+
11711216
#### [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py)
11721217

11731218
<img src=logos/urlhaus.png height=60>

doc/expansion/malwarebazaar.json

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"description": "Query the MALWAREbazaar API to get additional information about the input hash attribute.",
3+
"requirements": [],
4+
"input": "A hash attribute (md5, sha1 or sha256).",
5+
"output": "File object(s) related to the input attribute found on MALWAREbazaar databases.",
6+
"references": ["https://bazaar.abuse.ch/"],
7+
"features": "The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.\n\nThe module is using the new format of modules able to return object since the result is one or multiple MISP object(s)."
8+
}

doc/expansion/trustar_enrich.json

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"description": "Module to get enrich indicators with TruSTAR.",
3+
"logo": "logos/trustar.png",
4+
"input": "Any of the following MISP attributes:\n- btc\n- domain\n- email-src\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- url",
5+
"output": "MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level score and additional source and scoring info.",
6+
"references": ["https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html"],
7+
"features": "This module enriches MISP attributes with scoring and metadata from TruSTAR.\n\nThe TruSTAR indicator summary is appended to the attributes along with links to any associated reports."
8+
}

doc/logos/trustar.png

36.9 KB
Loading

misp_modules/modules/expansion/__init__.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from . import _vmray # noqa
22
import os
33
import sys
4+
45
sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
56

67
__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
@@ -15,5 +16,6 @@
1516
'qrcode', 'ocr_enrich', 'pdf_enrich', 'docx_enrich', 'xlsx_enrich', 'pptx_enrich',
1617
'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
1718
'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
18-
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb',
19-
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich']
19+
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
20+
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
21+
'trustar_enrich']

misp_modules/modules/expansion/circl_passivessl.py

+4
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ def parse(self):
4444
self.result = {'error': 'Not found'}
4545
return
4646

47+
if 'error' in results:
48+
self.result = {'error': results['error']}
49+
return
50+
4751
for ip_address, certificates in results.items():
4852
ip_uuid = self._handle_ip_attribute(ip_address)
4953
for certificate in certificates['certificates']:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import json
2+
import requests
3+
from pymisp import MISPEvent, MISPObject
4+
5+
mispattributes = {'input': ['md5', 'sha1', 'sha256'],
6+
'format': 'misp_standard'}
7+
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
8+
'description': 'Query Malware Bazaar to get additional information about the input hash.',
9+
'module-type': ['expansion', 'hover']}
10+
moduleconfig = []
11+
12+
13+
def parse_response(response):
14+
mapping = {'file_name': {'type': 'filename', 'object_relation': 'filename'},
15+
'file_size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
16+
'file_type_mime': {'type': 'mime-type', 'object_relation': 'mimetype'},
17+
'md5_hash': {'type': 'md5', 'object_relation': 'md5'},
18+
'sha1_hash': {'type': 'sha1', 'object_relation': 'sha1'},
19+
'sha256_hash': {'type': 'sha256', 'object_relation': 'sha256'},
20+
'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}}
21+
misp_event = MISPEvent()
22+
for data in response:
23+
misp_object = MISPObject('file')
24+
for feature, attribute in mapping.items():
25+
if feature in data:
26+
misp_attribute = {'value': data[feature]}
27+
misp_attribute.update(attribute)
28+
misp_object.add_attribute(**misp_attribute)
29+
misp_event.add_object(**misp_object)
30+
return {'results': {'Object': [json.loads(misp_object.to_json()) for misp_object in misp_event.objects]}}
31+
32+
33+
def handler(q=False):
34+
if q is False:
35+
return False
36+
request = json.loads(q)
37+
attribute = request['attribute']
38+
url = 'https://mb-api.abuse.ch/api/v1/'
39+
response = requests.post(url, data={'query': 'get_info', 'hash': attribute['value']}).json()
40+
query_status = response['query_status']
41+
if query_status == 'ok':
42+
return parse_response(response['data'])
43+
return {'error': 'Hash not found on MALWAREbazzar' if query_status == 'hash_not_found' else f'Problem encountered during the query: {query_status}'}
44+
45+
46+
def introspection():
47+
return mispattributes
48+
49+
50+
def version():
51+
moduleinfo['config'] = moduleconfig
52+
return moduleinfo

misp_modules/modules/expansion/rbl.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -88,18 +88,18 @@ def handler(q=False):
8888
else:
8989
misperrors['error'] = "Unsupported attributes type"
9090
return misperrors
91-
listed = []
92-
info = []
91+
listeds = []
92+
infos = []
9393
ipRev = '.'.join(ip.split('.')[::-1])
9494
for rbl in rbls:
9595
query = '{}.{}'.format(ipRev, rbl)
9696
try:
9797
txt = resolver.query(query, 'TXT')
98-
listed.append(query)
99-
info.append([str(t) for t in txt])
98+
listeds.append(query)
99+
infos.append([str(t) for t in txt])
100100
except Exception:
101101
continue
102-
result = "\n".join(["{}: {}".format(l, " - ".join(i)) for l, i in zip(listed, info)])
102+
result = "\n".join([f"{listed}: {' - '.join(info)}" for listed, info in zip(listeds, infos)])
103103
if not result:
104104
return {'error': 'No data found by querying known RBLs'}
105105
return {'results': [{'types': mispattributes.get('output'), 'values': result}]}

misp_modules/modules/expansion/sigma_queries.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'],
1313
'description': 'An expansion hover module to display the result of sigma queries.'}
1414
moduleconfig = []
15-
sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
15+
sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'mdatp', 'splunkxml', 'arcsight', 'qualys')
1616

1717

1818
def handler(q=False):
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import json
2+
import pymisp
3+
from pymisp import MISPAttribute, MISPEvent, MISPObject
4+
from trustar import TruStar
5+
6+
misperrors = {'error': "Error"}
7+
mispattributes = {
8+
'input': ["btc", "domain", "email-src", "filename", "hostname", "ip-src", "ip-dst", "malware-type", "md5", "sha1",
9+
"sha256", "url"], 'format': 'misp_standard'}
10+
11+
moduleinfo = {'version': "0.1", 'author': "Jesse Hedden",
12+
'description': "Enrich data with TruSTAR",
13+
'module-type': ["hover", "expansion"]}
14+
15+
moduleconfig = ["user_api_key", "user_api_secret", "enclave_ids"]
16+
17+
MAX_PAGE_SIZE = 100 # Max allowable page size returned from /1.3/indicators/summaries endpoint
18+
19+
20+
class TruSTARParser:
21+
ENTITY_TYPE_MAPPINGS = {
22+
'BITCOIN_ADDRESS': "btc",
23+
'CIDR_BLOCK': "ip-src",
24+
'CVE': "vulnerability",
25+
'URL': "url",
26+
'EMAIL_ADDRESS': "email-src",
27+
'SOFTWARE': "filename",
28+
'IP': "ip-src",
29+
'MALWARE': "malware-type",
30+
'MD5': "md5",
31+
'REGISTRY_KEY': "regkey",
32+
'SHA1': "sha1",
33+
'SHA256': "sha256"
34+
}
35+
36+
REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}"
37+
38+
CLIENT_METATAG = "MISP-{}".format(pymisp.__version__)
39+
40+
def __init__(self, attribute, config):
41+
config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',')
42+
config['client_metatag'] = self.CLIENT_METATAG
43+
self.ts_client = TruStar(config=config)
44+
45+
self.misp_event = MISPEvent()
46+
self.misp_attribute = MISPAttribute()
47+
self.misp_attribute.from_dict(**attribute)
48+
self.misp_event.add_attribute(**self.misp_attribute)
49+
50+
def get_results(self):
51+
"""
52+
Returns the MISP Event enriched with TruSTAR indicator summary data.
53+
"""
54+
event = json.loads(self.misp_event.to_json())
55+
results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
56+
return {'results': results}
57+
58+
def generate_trustar_links(self, entity_value):
59+
"""
60+
Generates links to TruSTAR reports if they exist.
61+
62+
:param entity_value: <str> Value of entity.
63+
"""
64+
report_links = list()
65+
trustar_reports = self.ts_client.search_reports(entity_value)
66+
for report in trustar_reports:
67+
report_links.append(self.REPORT_BASE_URL.format(report.id))
68+
69+
return report_links
70+
71+
def parse_indicator_summary(self, summaries):
72+
"""
73+
Converts a response from the TruSTAR /1.3/indicators/summaries endpoint
74+
a MISP trustar_report object and adds the summary data and links as attributes.
75+
76+
:param summaries: <generator> A TruSTAR Python SDK Page.generator object for generating
77+
indicator summaries pages.
78+
"""
79+
80+
for summary in summaries:
81+
trustar_obj = MISPObject('trustar_report')
82+
indicator_type = summary.indicator_type
83+
indicator_value = summary.value
84+
if indicator_type in self.ENTITY_TYPE_MAPPINGS:
85+
trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type],
86+
value=indicator_value)
87+
trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text",
88+
value=json.dumps(summary.to_dict(), sort_keys=True, indent=4))
89+
report_links = self.generate_trustar_links(indicator_value)
90+
for link in report_links:
91+
trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=link)
92+
self.misp_event.add_object(**trustar_obj)
93+
94+
95+
def handler(q=False):
96+
"""
97+
MISP handler function. A user's API key and secret will be retrieved from the MISP
98+
request and used to create a TruSTAR API client. If enclave IDs are provided, only
99+
those enclaves will be queried for data. Otherwise, all of the enclaves a user has
100+
access to will be queried.
101+
"""
102+
103+
if q is False:
104+
return False
105+
106+
request = json.loads(q)
107+
108+
config = request.get('config', {})
109+
if not config.get('user_api_key') or not config.get('user_api_secret'):
110+
misperrors['error'] = "Your TruSTAR API key and secret are required for indicator enrichment."
111+
return misperrors
112+
113+
attribute = request['attribute']
114+
trustar_parser = TruSTARParser(attribute, config)
115+
116+
try:
117+
summaries = list(
118+
trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE))
119+
except Exception as e:
120+
misperrors['error'] = "Unable to retrieve TruSTAR summary data: {}".format(e)
121+
return misperrors
122+
123+
trustar_parser.parse_indicator_summary(summaries)
124+
return trustar_parser.get_results()
125+
126+
127+
def introspection():
128+
return mispattributes
129+
130+
131+
def version():
132+
moduleinfo['config'] = moduleconfig
133+
return moduleinfo

misp_modules/modules/import_mod/csvimport.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -244,11 +244,11 @@ def __any_mandatory_misp_field(header):
244244

245245

246246
def __special_parsing(data, delimiter):
247-
return list(tuple(l.strip() for l in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
247+
return list(tuple(part.strip() for part in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
248248

249249

250250
def __standard_parsing(data):
251-
return list(tuple(l.strip() for l in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
251+
return list(tuple(part.strip() for part in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
252252

253253

254254
def handler(q=False):

misp_modules/modules/import_mod/threatanalyzer_import.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ def handler(q=False):
9999
results = process_analysis_json(json.loads(data.decode('utf-8')))
100100
except ValueError:
101101
log.warning('MISP modules {0} failed: uploaded file is not a zip or json file.'.format(request['module']))
102-
return {'error': 'Uploaded file is not a zip or json file.'.format(request['module'])}
102+
return {'error': 'Uploaded file is not a zip or json file.'}
103103
pass
104104
# keep only unique entries based on the value field
105105
results = list({v['values']: v for v in results}.values())

0 commit comments

Comments
 (0)