Merge pull request #397 from stratosphereips/develop

Slips v1.0.7
stratosphereips · Sep 15, 2023 · 822db6d · 822db6d
2 parents f938969 + fe19f88
commit 822db6d
Show file tree

Hide file tree

Showing 63 changed files with 2,178 additions and 280 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+-1.0.7 (September 2023):
+- CPU and memory profilers thanks to @danieltherealyang
+- Check dns queries and answers for whitelisted IPs and domains
+- Add AID flow hash to all conn.log flows, which is a combination of community_id and the flow's timestamp
+- Sqlite database improvements and better error handling
+- Add support for exporting Slips alerts to a sqlite db 
+
+
 -1.0.6 (June 2023):
 - Store flows in SQLite database in the output directory instead of redis.
 - 55% RAM usage decrease.

diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 <h1 align="center"> 
-Slips v1.0.6
+Slips v1.0.7
 </h1>
 
 [Documentation](https://stratospherelinuxips.readthedocs.io/en/develop/) — [Features](https://stratospherelinuxips.readthedocs.io/en/develop/features.html) — [Installation](#installation) — [Authors](#people-involved) — [Contributions](#contribute-to-slips)
@@ -32,7 +32,7 @@ Slips v1.0.6
 # Slips: Behavioral Machine Learning-Based Intrusion Prevention System
 
 
-Slips is a behavioral intrusion prevention system that uses machine learning to detect malicious behaviors in network traffic. Slips focus on targeted attacks, detection of command and control channels, and providing a good visualization for the analyst. It can analyze network traffic in real-time, network captures such as pcap files, and network flows produced by Suricata, Zeek/Bro, and Argus. Slips processes the input data, analyzes it, and highlights suspicious behavior that needs the analyst's attention.
+Slips is a powerful endpoint behavioral intrusion prevention and detection system that uses machine learning to detect malicious behaviors in network traffic. Slips can work with network traffic in real-time, pcap files, and network flows from popular tools like Suricata, Zeek/Bro, and Argus. Slips threat detection is based on a combination of machine learning models trained to detect malicious behaviors, 40+ threat intelligence feeds and expert heuristics. Slips gathers evidence of malicious behavior and uses extensively trained thresholds to trigger alerts when enough evidence is accumulated.
 
 <img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/slips.gif" width="850px"
 title="Slips in action.">

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.0.6
+1.0.7
diff --git a/config/slips.conf b/config/slips.conf
@@ -164,8 +164,9 @@ popup_alerts = no
 # [3] Generic Confs for the modules or to process the modules
 [modules]
 # List of modules to ignore. By default we always ignore the template! do not remove it from the list
-disable = [template, ensembling]
-# Names of other modules that you can disable: ensembling, threatintelligence, blocking,
+disable = [template, ensembling, rnnccdetection]
+# Names of other modules that you can disable (they all should be lowercase with no special characters):
+# ensembling, threatintelligence, blocking,
 #  networkdiscovery, timeline, virustotal, rnnccdetection, flowmldetection, updatemanager
 
 # For each line in timeline file there is a timestamp.
@@ -376,7 +377,45 @@ UID = 0
 GID = 0
 
 ####################
-# [11] enable or disable p2p for slips
+
+[Profiling]
+
+# [11] CPU profiling
+
+# enable cpu profiling [yes,no]
+cpu_profiler_enable = no
+
+# Available options are [dev,live]
+# dev for deterministic profiling. this will give precise information about the CPU usage
+# throughout the program runtime. This module cannot give live updates
+# live mode is for sampling data stream. To track the function stack in real time. it is accessible from web interface
+cpu_profiler_mode = dev
+
+# profile all subprocesses in dev mode [yes,no].
+cpu_profiler_multiprocess = yes
+
+# set number of tracer entries (dev mode only)
+cpu_profiler_dev_mode_entries = 1000000
+
+# set maximum output lines (live mode only)
+cpu_profiler_output_limit = 20
+
+# set the wait time between sampling sequences in seconds (live mode only)
+cpu_profiler_sampling_interval = 20
+
+# [12] Memory Profiling
+
+# enable memory profiling [yes,no]
+memory_profiler_enable = no
+
+# set profiling mode [dev,live]
+memory_profiler_mode = live
+
+# profile all subprocesses [yes,no]
+memory_profiler_multiprocess = yes
+
+####################
+# [13] enable or disable p2p for slips
 [P2P]
 
 # create p2p.log with additional info about peer communications? yes or no

diff --git a/conftest.py b/conftest.py
@@ -19,7 +19,7 @@
 
 
 @pytest.fixture
-def mock_db():
+def mock_rdb():
     # Create a mock version of the database object
     with patch('slips_files.core.database.database_manager.DBManager') as mock:
         yield mock.return_value

diff --git a/docker/macosm1-image/requirements-macos-m1-docker.txt b/docker/macosm1-image/requirements-macos-m1-docker.txt
@@ -36,4 +36,8 @@ wheel
 flask
 tld
 tqdm
-termcolor
+communityid
+termcolor
+memray
+viztracer
+yappi
diff --git a/docs/features.md b/docs/features.md
@@ -1017,8 +1017,47 @@ and ICMP-AddressMaskScan based on the icmp type
 We detect a scan every threshold. So we generate an evidence when there is 
 5,10,15, .. etc. ICMP established connections to different IPs.
 
+### CPU Profiling
 
+Slips is shipped with its own tool for CPU Profiling, it can be found it ```slips_files/common/cpu_profiler.py```
 
+CPU Profiling supports 2 modes: live and development mode
+
+#### Live mode:
+The main purpose of this mode it to show live CPU stats in the web interface.
+"live" mode publishes updates during the runtime of the program to the redis channel 'cpu_profile' so that the web interface can use them
+
+#### Development mode:
+
+ Setting the mode to "dev" outputs a JSON file of the CPU usage at the end of the program run.
+ It is recommended to only use dev mode for static file inputs (pcaps, suricata files, binetflows, etc.) instead of interface and growing zeek dirs, because longer runs result in profiling data loss and not everything will get recorded.
+The JSON file created in this mode is placed in the output dir of the current run and can be viewed by running the following command
+
+```vizviewer results.json``` 
+
+then going to http://127.0.0.1:9001/ in your browser for seeing the visualizations of the CPU usage
+
+
+Options to enable cpu profiling can be found under the [Profiling] section of the ```slips.conf``` file.
+```cpu_profiler_enable``` set to "yes" enables cpu profiling, or "no" to disable it.
+```cpu_profiler_mode``` can be set to "live" or "dev". Setting to 
+```cpu_profiler_multiprocess``` can be set to "yes" or "no" and only affects the dev mode profiling. If set to "yes" then all processes will be profiled. If set to "no" then only the main process (slips.py) will be profiled.
+```cpu_profiler_output_limit``` is set to an integer value and only affects the live mode profiling. This option sets the limit on the number of processes output for live mode profiling updates.
+```cpu_profiler_sampling_interval``` is set to an integer value and only affects the live mode profiling. This option sets the duration in seconds of live mode sampling intervals. It is recommended to set this option greater than 10 seconds otherwise there won't be much useful information captured during sampling.
+
+### Memory Profiling
+Memory profiling can be found in ```slips_files/common/memory_profiler.py```
+
+Just like CPU profiling, it also has supports live and development mode.
+Set ```memory_profiler_enable``` to ```yes``` to enable this feature.
+Set ```memory_profiler_mode``` to ```live``` to use live mode or ```dev``` to use development mode profiling.
+
+#### Live Mode
+This mode shows memory usage stats during the runtime of the program.
+```memory_profiler_multiprocess``` controls whether live mode tracks all processes or only the main process. If set to no, the program will wait for you to connect from a different terminal using the command ```memray live <port_number>```, where port_number is 5000 by default. After connection, the program will continue with its run and the terminal that is connected will receive a feed of the memory statistics. If set to yes, the redis channel "memory_profile" can be used to set pid of the process to be tracked. Only a single process can be tracked at a time. The interface is cumbersome to use from the command line so multiprocess live profiling is intended to be used primarily from the web interface.
+
+#### Development Mode
+When enabled, the profiler will output the profile data into the output directory. The data will be in the ```memoryprofile``` directory of the output directory of the run. Each process during the run of the program will have an associated binary file. Each of the generated binaries will automatically be converted to viewable html files, with each process converted to a flamegraph and table format. All generated files will be denoted by their PID.
 
 ---
 

diff --git a/docs/images/alternate_mem_profiler_testing.png b/docs/images/alternate_mem_profiler_testing.png
diff --git a/docs/images/cpu-profiler-config.png b/docs/images/cpu-profiler-config.png
diff --git a/docs/images/cpu-profiler-live-mode.png b/docs/images/cpu-profiler-live-mode.png
diff --git a/docs/images/cpu-profiler-live-results.png b/docs/images/cpu-profiler-live-results.png
diff --git a/docs/images/cpu-profiler-starting.png b/docs/images/cpu-profiler-starting.png
diff --git a/docs/images/cpu-profiler-termination.png b/docs/images/cpu-profiler-termination.png
diff --git a/docs/images/cpu=profiler-results.png b/docs/images/cpu=profiler-results.png
diff --git a/docs/images/flamegraph.png b/docs/images/flamegraph.png
diff --git a/docs/images/live-mem-profiler.png b/docs/images/live-mem-profiler.png
diff --git a/docs/images/mem-profiler-ending.png b/docs/images/mem-profiler-ending.png
diff --git a/docs/images/mem-profiler-running.png b/docs/images/mem-profiler-running.png
diff --git a/docs/images/mem-profiler-starting.png b/docs/images/mem-profiler-starting.png
diff --git a/docs/images/mem-profiler-table-view.png b/docs/images/mem-profiler-table-view.png
diff --git a/docs/images/memory_multi_process_profiler_structure.png b/docs/images/memory_multi_process_profiler_structure.png
diff --git a/docs/images/memory_profiler_interface.png b/docs/images/memory_profiler_interface.png
diff --git a/docs/images/memory_profiler_structure.png b/docs/images/memory_profiler_structure.png
diff --git a/docs/images/running-vizviewer.png b/docs/images/running-vizviewer.png
diff --git a/docs/images/slips.gif b/docs/images/slips.gif
diff --git a/docs/images/testing_live_cpu_profiler.png b/docs/images/testing_live_cpu_profiler.png
diff --git a/docs/images/testinig_mem_profiler_live_mode.png b/docs/images/testinig_mem_profiler_live_mode.png
diff --git a/docs/images/testinig_mem_profiler_live_mode_step3_1.png b/docs/images/testinig_mem_profiler_live_mode_step3_1.png
diff --git a/docs/images/testinig_mem_profiler_live_mode_step3_2.png b/docs/images/testinig_mem_profiler_live_mode_step3_2.png