diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..9f223bb --- /dev/null +++ b/Pipfile @@ -0,0 +1,44 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +asn1crypto = "==0.24.0" +cached-property = "==1.5.1" +certifi = "==2018.8.24" +cffi = "==1.11.5" +chardet = "==3.0.4" +click = "==7.0" +cryptography = "==2.3.1" +defusedxml = "==0.5.0" +dnspython = "==1.15.0" +exchangelib = "==1.12.0" +flask = "==1.0.2" +future = "==0.16.0" +idna = "==2.7" +isodate = "==0.6.0" +itsdangerous = "==0.24" +jinja2 = "==2.10" +lxml = "==4.2.5" +markupsafe = "==1.0" +ntlm-auth = "==1.2.0" +pycparser = "==2.19" +pygments = "==2.2.0" +python-dateutil = "==2.7.3" +python-magic = "==0.4.15" +pytz = "==2018.5" +requests-ntlm = "==1.1.0" +requests = "==2.20.0" +six = "==1.11.0" +python-slugify = "==1.2.6" +thehive4py = "==1.5.1" +tzlocal = "==1.5.1" +urllib3 = "==1.23" +werkzeug = "==0.14.1" + +[dev-packages] + +[requires] +python_version = "3.11" +python_full_version = "3.11.4" diff --git a/README.md b/README.md index a401ff2..0b2efc1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ +## Synpase for TheHive5 + +

+ +

+ +> **_NOTE:_** The work is in progress to update **Ews2Case**. In this wave of updates, we were interested in updating **QRadar2Alert** (pulling offenses and create alerts to Thehive5). + +- [x] Project in progress + Synapse is a free, open source meta alert feeder that allows you to feed [TheHive](https://github.com/TheHive-Project/TheHive) from multiple alert sources at once. It leverages TheHive's API to automate case and alert creation. Thanks to Synapse, you can swiftly create cases or alerts in TheHive out of email notifications or SIEM events. @@ -7,6 +18,8 @@ Currently, Synapse supports the following alert sources: - Microsoft O365 - IBM QRadar +> We have updated Synapse to support theHive 5.x REST API + # Overview Most of the time, transforming a security event or a notification about a suspicious email requires several actions and conditions. Synapse gathers those into workflows. @@ -29,7 +42,10 @@ The [user guide](docs/user_guide.md) should contain all the information you need 2. Fill in the config file 3. Execute: ```python3 app.py``` -While all operating systems running Python 3 can be used for Synapse, we recommend the use of Ubuntu. +### Test environment +The new update was tested on: +* Python 3.11.4 +* Ubuntu/Kali Linux /Redhat # License Synapse is an open source and free software released under the diff --git a/app.py b/app.py index 49b7bbc..aaf6332 100644 --- a/app.py +++ b/app.py @@ -6,7 +6,7 @@ from flask import Flask, request, jsonify from workflows.common.common import getConf -from workflows.Ews2Case import connectEws +#from workflows.Ews2Case import connectEws from workflows.QRadar2Alert import allOffense2Alert from workflows.ManageWebhooks import manageWebhook @@ -33,6 +33,10 @@ app = Flask(__name__) +@app.route('/', methods=['GET']) +def index(): + return "Synapse is up and running." + @app.route('/webhook', methods=['POST']) def listenWebhook(): if request.is_json: @@ -57,26 +61,35 @@ def ews2case(): else: return jsonify(workflowReport), 500 + @app.route('/QRadar2alert', methods=['POST']) def QRadar2alert(): + # Get token from config file conf/synpase.conf + Token = getConf().get('Token','auth_token') if request.is_json: content = request.get_json() - if 'timerange' in content: - workflowReport = allOffense2Alert(content['timerange']) - if workflowReport['success']: - return jsonify(workflowReport), 200 + if content['token'] == Token: + if 'timerange' in content: + workflowReport = allOffense2Alert(content['timerange']) + if workflowReport['success']: + return jsonify(workflowReport), 200 + else: + return jsonify(workflowReport), 500 else: - return jsonify(workflowReport), 500 + logger.error('Missing key/value') + return jsonify({'sucess':False, 'message':"timerange key missing in request"}), 500 else: - logger.error('Missing key/value') - return jsonify({'sucess':False, 'message':"timerange key missing in request"}), 500 + logger.error('Missing token!') + return jsonify({"message":"Missing a valid token !"}), 403 + else: logger.error('Not json request') return jsonify({'sucess':False, 'message':"Request didn't contain valid JSON"}), 400 + @app.route('/version', methods=['GET']) def getSynapseVersion(): - return jsonify({'version': '1.1.1'}), 200 + return jsonify({'version': '1.1.2'}), 200 if __name__ == '__main__': cfg = getConf() @@ -84,4 +97,4 @@ def getSynapseVersion(): host=cfg.get('api', 'host'), port=cfg.get('api', 'port'), threaded=cfg.get('api', 'threaded') - ) + ) \ No newline at end of file diff --git a/conf/synapse.conf b/conf/synapse.conf index e1ad5fb..13deaf7 100644 --- a/conf/synapse.conf +++ b/conf/synapse.conf @@ -28,3 +28,6 @@ server:qradar.stargazer.org auth_token:d6e-8f-4e-85-55738fd cert_filepath:/home/dc/qradar.crt api_version:8.0 + +[Token] +auth_token=CHANGE_ME diff --git a/docs/user_guide.md b/docs/user_guide.md index 73a2e5b..9613f85 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -14,6 +14,7 @@ This guide will go through installation and basic configuration for Synapse. + [Stopping the application](#stopping-the-application) + [Starting the application](#starting-the-application) + [Logs](#logs) + + [Crontab](#crontab) + [Update](#update) ## Installation @@ -27,6 +28,16 @@ sudo apt install python3-dev libkrb5-dev gcc sudo pip3 install -r requirements.txt ``` +#### Thehive4py +Install the new thehive4py library +``` +git clone https://github.com/TheHive-Project/TheHive4py +cd TheHive4py +python3 setup.py install +## If failed, edit setup.py version=parse_version() +version="5" +``` + ## Configuration ### Synapse user @@ -69,6 +80,18 @@ url:http://127.0.0.1:9000 user:synapse api_key:r4n0O8SvEll/VZdOD8r0hZneOWfOmth6 ``` +### [Token] section + +Create a new secure token that will be used to invoke QRadar2Alert function +``` +tr -dc A-Za-z0-9 > /home/Synpase/logs/synapse_curl.log 2>&1 +``` + # Update In order to update Synapse (minor version), just pull the new version from Github and run the application: diff --git a/examples/qradar2alert_post.sh b/examples/qradar2alert_post.sh index 58c0975..2e14866 100755 --- a/examples/qradar2alert_post.sh +++ b/examples/qradar2alert_post.sh @@ -1 +1,2 @@ -curl --header "Content-Type: application/json" --request POST --data '{"timerange":10}' http://127.0.0.1:5000/QRadar2alert +# Trigger FUNC (Change the token with the a valid token in conf/synpase.conf) +curl -H "Content-Type: application/json" -XPOST -d '{"timerange":10,"token":"CHANGE_ME"}' http://127.0.0.1:5000/QRadar2alert diff --git a/requirements.txt b/requirements.txt index 885928c..8b0de4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,33 +1,34 @@ --i https://pypi.org/simple -asn1crypto==0.24.0 -cached-property==1.5.1 -certifi==2018.8.24 -cffi==1.11.5 -chardet==3.0.4 -click==7.0 -cryptography==2.3.1 -defusedxml==0.5.0 -dnspython==1.15.0 -exchangelib==1.12.0 -flask==1.0.2 -future==0.16.0 -idna==2.7 -isodate==0.6.0 -itsdangerous==0.24 -jinja2==2.10 -lxml==4.2.5 -markupsafe==1.0 -ntlm-auth==1.2.0 -pycparser==2.19 -pygments==2.2.0 -python-dateutil==2.7.3 -python-magic==0.4.15 -pytz==2018.5 -requests-ntlm==1.1.0 -requests==2.20.0 -six==1.11.0 -python-slugify==1.2.6 -thehive4py==1.5.1 -tzlocal==1.5.1 -urllib3==1.23 -werkzeug==0.14.1 +anyio==3.7.1 +blinker==1.6.2 +cached-property==1.5.2 +certifi==2023.5.7 +cffi==1.15.1 +charset-normalizer==3.2.0 +click==8.1.6 +cryptography==41.0.2 +defusedxml==0.7.1 +dnspython==2.4.0 +exchangelib==5.0.3 +Flask==2.3.2 +h11==0.14.0 +httpcore==0.17.3 +idna==3.4 +isodate==0.6.1 +itsdangerous==2.1.2 +Jinja2==3.1.2 +lxml==4.9.3 +MarkupSafe==2.1.3 +oauthlib==3.2.2 +pycparser==2.21 +Pygments==2.15.1 +pyspnego==0.9.1 +requests==2.31.0 +requests-ntlm==1.2.0 +requests-oauthlib==1.3.1 +six==1.16.0 +sniffio==1.3.0 +thehive4py==2.0.0b4 +tzdata==2023.3 +tzlocal==5.0.1 +urllib3==2.0.4 +Werkzeug==2.3.6 diff --git a/workflows/QRadar2Alert.py b/workflows/QRadar2Alert.py index 1edf9ae..7c2c86e 100644 --- a/workflows/QRadar2Alert.py +++ b/workflows/QRadar2Alert.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf8 -*- +# Synapse that support Thehive 5.0 Rest API +# Modified by @ihebski import os, sys import logging import copy import json -from time import sleep +from time import sleep,time current_dir = os.path.dirname(os.path.abspath(__file__)) app_dir = current_dir + '/..' @@ -21,7 +23,6 @@ def getEnrichedOffenses(qradarConnector, timerange): for offense in qradarConnector.getOffenses(timerange): enrichedOffenses.append(enrichOffense(qradarConnector, offense)) - return enrichedOffenses def enrichOffense(qradarConnector, offense): @@ -122,32 +123,38 @@ def getHiveSeverity(offense): 'uri_path', 'url', 'user-agent'] + # Create data for Observable field + # format : "observables": [{ "dataType": "ip", "data": "127.0.0.1",'tags': ['dst'],'message': 'Local destination IP'}] artifacts = [] for artifact in offense['artifacts']: if artifact['dataType'] in defaultObservableDatatype: - hiveArtifact = theHiveConnector.craftAlertArtifact(dataType=artifact['dataType'], data=artifact['data'], message=artifact['message'], tags=artifact['tags']) + # create JSON for observable field + hiveArtifact = {'data': artifact['data'], 'dataType': artifact['dataType'], 'message': artifact['message'], 'tags': artifact['tags']} + else: tags = list() tags.append('type:' + artifact['dataType']) - hiveArtifact = theHiveConnector.craftAlertArtifact(dataType='other', data=artifact['data'], message=artifact['message'], tags=tags) + hiveArtifact = {'data': artifact['data'], 'dataType': 'other', 'message': artifact['message'], 'tags': tags} artifacts.append(hiveArtifact) - + + # Build TheHive alert + # return JSON of the alert /api/v1/docs/index.html#tag/Alert/operation/Create%20Alert + # FUNC: craftAlert(title, description, severity, date, tags, tlp, status, type, source,sourceRef, artifacts, caseTemplate) alert = theHiveConnector.craftAlert( offense['description'], craftAlertDescription(offense), getHiveSeverity(offense), offense['start_time'], tags, - 2, + 1, 'Imported', 'internal', 'QRadar_Offenses', str(offense['id']), artifacts, - '') - + 'Generic Template Qradar') return alert @@ -180,14 +187,15 @@ def allOffense2Alert(timerange): q['sourceRef'] = str(offense['id']) logger.info('Looking for offense %s in TheHive alerts', str(offense['id'])) results = theHiveConnector.findAlert(q) - if len(results) == 0: + + if results == 0: logger.info('Offense %s not found in TheHive alerts, creating it', str(offense['id'])) offense_report = dict() enrichedOffense = enrichOffense(qradarConnector, offense) try: theHiveAlert = qradarOffenseToHiveAlert(theHiveConnector, enrichedOffense) - theHiveEsAlertId = theHiveConnector.createAlert(theHiveAlert)['id'] + theHiveEsAlertId = theHiveConnector.createAlert(theHiveAlert) offense_report['raised_alert_id'] = theHiveEsAlertId offense_report['qradar_offense_id'] = offense['id'] @@ -210,7 +218,6 @@ def allOffense2Alert(timerange): logger.info('Offense %s already imported as alert', str(offense['id'])) except Exception as e: - logger.error('Failed to create alert from QRadar offense (retrieving offenses failed)', exc_info=True) report['success'] = False report['message'] = "%s: Failed to create alert from offense" % str(e) @@ -250,7 +257,9 @@ def craftAlertDescription(offense): return description + if __name__ == '__main__': #hardcoding timerange as 1 minute when not using the API timerange = 1 - offense2Alert(timerange) + offense2Alert(timerange) + diff --git a/workflows/objects/TheHiveConnector.py b/workflows/objects/TheHiveConnector.py index 630be09..2b960de 100644 --- a/workflows/objects/TheHiveConnector.py +++ b/workflows/objects/TheHiveConnector.py @@ -4,9 +4,23 @@ import logging import json -from thehive4py.api import TheHiveApi -from thehive4py.models import Case, CaseTask, CaseTaskLog, CaseObservable, AlertArtifact, Alert -from thehive4py.query import Eq +from thehive4py.client import TheHiveApi +from thehive4py.client import TheHiveApi +from thehive4py.errors import TheHiveError +from thehive4py.helpers import now_to_ts +from thehive4py.query.filters import Eq +from thehive4py.query.sort import Asc +from thehive4py.types.alert import OutputAlert +from thehive4py.types.case import ( + CaseStatus, + ImpactStatus, + InputBulkUpdateCase, + InputUpdateCase, + OutputCase, +) +from thehive4py.types.observable import InputObservable +from thehive4py.types.share import InputShare + class TheHiveConnector: 'TheHive connector' @@ -56,39 +70,25 @@ def searchCaseByDescription(self, string): #unknown use case raise ValueError('unknown use case after searching case by description') - def craftCase(self, title, description): self.logger.info('%s.craftCase starts', __name__) - - case = Case(title=title, - tlp=2, - tags=['Synapse'], - description=description, - ) - - return case + json_case = {'title':title,'description':description,'tags':['Synapse'],'severity':1} + return json_case def createCase(self, case): self.logger.info('%s.createCase starts', __name__) - - response = self.theHiveApi.create_case(case) - - if response.status_code == 201: - esCaseId = response.json()['id'] - createdCase = self.theHiveApi.case(esCaseId) - return createdCase + created_case = self.theHiveApi.case.create(case) + fetched_case = self.theHiveApi.case.get(created_case["_id"]) + if created_case == fetched_case: + return created_case else: self.logger.error('Case creation failed') - raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True)) + raise ValueError(json.dumps({'message':'Failed to Create a new Case !'})) def assignCase(self, case, assignee): self.logger.info('%s.assignCase starts', __name__) - - esCaseId = case.id - case.owner = assignee - self.theHiveApi.update_case(case) - - updatedCase = self.theHiveApi.case(esCaseId) + update_fields: InputUpdateCase = {'assignee':assignee} + updatedCase = self.theHiveApi.case.update(case_id=case['_id'], case=update_fields) return updatedCase def craftCommTask(self): @@ -112,12 +112,6 @@ def createTask(self, esCaseId, task): self.logger.error('Task creation failed') raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True)) - def craftAlertArtifact(self, **attributes): - self.logger.info('%s.craftAlertArtifact starts', __name__) - - alertArtifact = AlertArtifact(dataType=attributes["dataType"], message=attributes["message"], data=attributes["data"], tags=attributes['tags']) - - return alertArtifact def craftTaskLog(self, textLog): self.logger.info('%s.craftTaskLog starts', __name__) @@ -128,15 +122,13 @@ def craftTaskLog(self, textLog): def addTaskLog(self, esTaskId, textLog): self.logger.info('%s.addTaskLog starts', __name__) - - response = self.theHiveApi.create_task_log(esTaskId, textLog) - - if response.status_code == 201: - esCreatedTaskLogId = response.json()['id'] - return esCreatedTaskLogId + response = self.theHiveApi.task_log.create(task_id=esTaskId,task_log={"message": textLog}) + fetched_log = self.theHiveApi.task_log.get(task_log_id=response["_id"]) + if response == fetched_log: + return response else: self.logger.error('Task log creation failed') - raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True)) + raise ValueError(json.dumps({'message':'Task log creation failed'})) def getTaskIdByTitle(self, esCaseId, taskTitle): self.logger.info('%s.getTaskIdByName starts', __name__) @@ -173,52 +165,54 @@ def addFileObservable(self, esCaseId, filepath, comment): def craftAlert(self, title, description, severity, date, tags, tlp, status, type, source, sourceRef, artifacts, caseTemplate): self.logger.info('%s.craftAlert starts', __name__) - - alert = Alert(title=title, - description=description, - severity=severity, - date=date, - tags=tags, - tlp=tlp, - type=type, - source=source, - sourceRef=sourceRef, - artifacts=artifacts, - caseTemplate=caseTemplate) - - return alert + json_alert = { + "type": type, + "source": source, + "sourceRef": sourceRef, + "title": title, + "description": description, + "severity": severity, + "date": date, + "status": "New", + "tags": tags, + "tlp": tlp, + "caseTemplate": caseTemplate, + "observables": artifacts} + + return json_alert def createAlert(self, alert): self.logger.info('%s.createAlert starts', __name__) + self.logger.info(alert) + try: + response = self.theHiveApi.alert.create(alert) + except Exception as e: + if 'CreateError' in str(e): + self.logger.info('%s.createAlert starts failed', __name__) + else: + return response - response = self.theHiveApi.create_alert(alert) - - if response.status_code == 201: - return response.json() - else: - self.logger.error('Alert creation failed') - raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True)) def findAlert(self, q): """ Search for alerts in TheHive for a given query :param q: TheHive query - :type q: dict - - :return results: list of dict, each dict describes an alert - :rtype results: list + :return : JSON """ self.logger.info('%s.findAlert starts', __name__) - response = self.theHiveApi.find_alerts(query=q) - if response.status_code == 200: - results = response.json() + response = self.theHiveApi.alert.find(q) + self.logger.info(q) + self.logger.info(response) + if len(response) > 0: + response = response[0] + results = response return results else: self.logger.error('findAlert failed') - raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True)) + return 0 def findFirstMatchingTemplate(self, searchstring): self.logger.info('%s.findFirstMatchingTemplate starts', __name__) @@ -232,4 +226,4 @@ def findFirstMatchingTemplate(self, searchstring): if searchstring in template['name']: return template - return None + return None \ No newline at end of file