Merge pull request #482 from stratosphereips/develop

Slips v1.0.12

CHANGELOG.md

@@ -1,3 +1,20 @@
+- 1.0.12 (March 2024)
+- Add an option to specify the current client IP in slips.conf to help avoid false positives.
+- Better handling of URLhaus threat intelligence.
+- Change how slips determines the local network of the current client IP.
+- Fix issues with the progress bar.
+- Fix problem logging alerts and errors to alerts.log and erros.log.
+- Fix problem reporting evidence to other peers.
+- Fix problem starting the web interface.
+- Fix whitelists.
+- Improve how the evidence for young domain detections is set. 
+- Remove the description of blacklisted IPs from the evidence description and add the source TI feed instead.
+- Set evidence to all young domain IPs when a connection to a young domain is found.
+- Set two evidence in some detections e.g. when the source address connects to a blacklisted IP, evidence is set for both.
+- Use blacklist name instead of IP description in all evidence. 
+- Use the latest Redis and NodeJS version in all docker images.
+
+
 - 1.0.11 (February 2024)
 - Improve the logging of evidence in alerts.json and alerts.log.
 - Optimize the storing of evidence in the Redis database.

README.md

@@ -1,5 +1,5 @@
 <h1 align="center"> 
-Slips v1.0.11
+Slips v1.0.12
 </h1>
 
 

VERSION

@@ -1 +1 @@
-1.0.11
+1.0.12

config/slips.conf

@@ -137,6 +137,14 @@ export_labeled_flows = no
 # export_format can be tsv or json. this parameter is ignored if export_labeled_flows is set to no
 export_format = json
 
+# These are the IPs that we see the majority of traffic going out of from.
+# for example, this can be your own IP or some computer you’re monitoring
+# when using slips on an interface, this client IP is automatically set as
+# your own IP and is used to improve detections
+# it would be useful to specify it when analyzing pcaps or zeek logs
+#client_ips = [10.0.0.1, 172.16.0.9, 172.217.171.238]
+client_ips = []
+
 #####################
 # [2] Configuration for the detections
 [detection]

docker/P2P-image/Dockerfile

@@ -18,23 +18,27 @@ RUN apt update && apt install -y --no-install-recommends \
     curl \
     gnupg \
     nano \
+    lsb-release \
  && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \
- && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+ && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \
+ && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
+ && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list
+
 
 # Install Slips dependencies.
-RUN apt update && apt install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
-    redis-server \
     zeek \
+    redis \
     python3-pip \
     python3-certifi \
     python3-dev \
     build-essential \
     file \
     lsof \
     net-tools \
-    iproute2 \
     iptables \
+    iproute2 \
     python3-tzlocal \
     nfdump \
     tshark \
@@ -68,7 +72,7 @@ RUN pip3 install -r install/requirements.txt
 
 
 # For Kalipso:
-RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs
+RUN curl -fsSL https://deb.nodesource.com/setup_21.x |  bash -  && apt install -y --no-install-recommends nodejs
 
 # Switch to kalipso dir to install node dependencies
 WORKDIR ${SLIPS_DIR}/modules/kalipso

docker/dependency-image/Dockerfile

@@ -31,23 +31,27 @@ RUN apt update && apt install -y --no-install-recommends \
     git \
     curl \
     gnupg \
+    lsb-release \
  && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \
- && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+ && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \
+ && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
+ && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list
+
 
 # Install Slips dependencies.
-RUN apt update && apt install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
-    redis-server \
     zeek \
+    redis \
     python3-pip \
     python3-certifi \
     python3-dev \
     build-essential \
     file \
     lsof \
     net-tools \
-    iproute2 \
     iptables \
+    iproute2 \
     python3-tzlocal \
     nfdump \
     tshark \
@@ -58,6 +62,7 @@ RUN apt update && apt install -y --no-install-recommends \
  && ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
 
 
+
 # Install python dependencies
 
 # you should build the image using

docker/macosm1-P2P-image/Dockerfile

@@ -18,23 +18,27 @@ RUN apt update && apt install -y --no-install-recommends \
     git \
     curl \
     gnupg \
+    lsb-release \
  && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \
- && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+ && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \
+ && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
+ && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list
+
 
 # Install Slips dependencies.
-RUN apt update && apt install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
-    redis-server \
     zeek \
+    redis \
     python3-pip \
     python3-certifi \
     python3-dev \
     build-essential \
     file \
     lsof \
     net-tools \
-    iproute2 \
     iptables \
+    iproute2 \
     python3-tzlocal \
     nfdump \
     tshark \
@@ -64,7 +68,7 @@ RUN pip3 install --upgrade pip
 RUN pip3 install -r ${SLIPS_DIR}/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt
 
 # For Kalipso:
-RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs
+RUN curl -fsSL https://deb.nodesource.com/setup_21.x |  bash -  && apt install -y --no-install-recommends nodejs
 
 # Switch to kalipso dir to install node dependencies
 WORKDIR ${SLIPS_DIR}/modules/kalipso

docker/macosm1-P2P-image/requirements-macos-m1-docker.txt

@@ -23,7 +23,7 @@ pytest-mock
 pytest-xdist
 slackclient
 scipy
-sklearn
+scikit-learn
 GitPython
 protobuf
 blinker

docker/macosm1-image/Dockerfile

@@ -18,23 +18,27 @@ RUN apt update && apt install -y --no-install-recommends \
     git \
     curl \
     gnupg \
+    lsb-release \
  && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \
- && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+ && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \
+ && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
+ && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list
+
 
 # Install Slips dependencies.
-RUN apt update && apt install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
-    redis-server \
     zeek \
+    redis \
     python3-pip \
     python3-certifi \
     python3-dev \
     build-essential \
     file \
     lsof \
     net-tools \
-    iproute2 \
     iptables \
+    iproute2 \
     python3-tzlocal \
     nfdump \
     tshark \
@@ -60,7 +64,7 @@ RUN pip3 install --upgrade pip
 RUN pip3 install -r ${SLIPS_DIR}/docker/macosm1-image/requirements-macos-m1-docker.txt
 
 # For Kalipso:
-RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs
+RUN curl -fsSL https://deb.nodesource.com/setup_21.x |  bash -  && apt install -y --no-install-recommends nodejs
 
 # Switch to kalipso dir to install node dependencies
 WORKDIR ${SLIPS_DIR}/modules/kalipso

docker/tensorflow-image/Dockerfile

@@ -16,14 +16,17 @@ RUN apt update && apt install -y --no-install-recommends \
     git \
     curl \
     gnupg \
+    lsb-release \
  && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \
- && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+ && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \
+ && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
+ && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list
+
 
 # Install Slips dependencies.
-RUN apt update && apt install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
-    curl \
-    redis-server \
+    redis \
     zeek \
     python3-pip \
     python3-certifi \
@@ -70,7 +73,7 @@ RUN pip3 install -r ${SLIPS_DIR}/install/requirements.txt
 
 
 # For Kalipso:
-RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs
+RUN curl -fsSL https://deb.nodesource.com/setup_21.x |  bash -  && apt install -y --no-install-recommends nodejs
 
 # Switch to kalipso dir to install node dependencies
 WORKDIR ${SLIPS_DIR}/modules/kalipso

docker/ubuntu-image/Dockerfile

@@ -9,30 +9,34 @@ ENV IS_IN_A_DOCKER_CONTAINER True
 # destionation dir for slips inside the container
 ENV SLIPS_DIR /StratosphereLinuxIPS
 
-# Install wget and add Zeek repository to our sources.
+# Install wget and add Zeek and redis repositories to our sources.
 RUN apt update && apt install -y --no-install-recommends \
     wget \
     ca-certificates \
     git \
     curl \
     gnupg \
+    lsb-release \
  && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | tee /etc/apt/sources.list.d/security:zeek.list \
- && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+ && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/security_zeek.gpg \
+ && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
+ && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list
+
 
 # Install Slips dependencies.
-RUN apt update && apt install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
-    redis-server \
     zeek \
+    redis \
     python3-pip \
     python3-certifi \
     python3-dev \
     build-essential \
     file \
     lsof \
     net-tools \
-    iproute2 \
     iptables \
+    iproute2 \
     python3-tzlocal \
     nfdump \
     tshark \
@@ -53,7 +57,7 @@ RUN pip3 install --no-cache-dir -r ${SLIPS_DIR}/install/requirements.txt
 
 
 # For Kalipso:
-RUN curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs
+RUN curl -fsSL https://deb.nodesource.com/setup_21.x |  bash -  && apt install -y --no-install-recommends nodejs
 
 # Switch to kalipso dir to install node dependencies
 WORKDIR ${SLIPS_DIR}/modules/kalipso

docs/flowalerts.md

@@ -319,6 +319,17 @@ For example if the currently used local network is: 192.168.1.0/24
 
 and slips sees a forged packet going from 192.168.1.2 to 10.0.0.1, it will alert
 
+Slips detects the current local network by using the local network of the private
+ips specified in ```client_ips``` parameter in ```slips.conf```
+
+If no IPs are specified, slips uses the local network of the first private source ip 
+found in the traffic.
+
+This threat level of this detection is low if the source ip is the one outside of local network 
+because it's unlikely.
+and high if the destination ip is the one outside of local network. 
+
+
 ## High entropy DNS TXT answers 
 
 Slips check every DNS answer with TXT record for high entropy

docs/installation.md

@@ -294,7 +294,7 @@ _Note: for those using a different base image, you need to also install tensorfl
 
 As we mentioned before, the GUI of Slips known as Kalipso relies on NodeJs v19. Make sure to use NodeJs greater than version 12. For Kalipso to work, we will install the following npm packages:
 
-    curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt install -y --no-install-recommends nodejs
+    curl -fsSL https://deb.nodesource.com/setup_21.x |  sudo -E bash - && sudo apt install -y --no-install-recommends nodejs
     cd modules/kalipso &&  npm install
 
 ####  Installing Zeek

docs/usage.md

@@ -248,7 +248,7 @@ There are two options how to run Kalipso Locally:
 
 You can run Kalipso as a shell script in another terminal using the command:
 
-	```./kalipso.sh```
+	./kalipso.sh
 
 
 In docker, you can open a new terminal inside the slips container and execute ```./kalipso.sh```
@@ -276,14 +276,14 @@ The traffic of IP is splitted into time windows. each time window is 1h long of
 
 You can press Enter of any of them to view the list of flows in the timewindow.
 
-<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/web_interface.png" width="850px"
+<img src="https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/docs/images/web_interface.png" width="850px">
 
 You can switch to the flows view in kalipso by pressing TAB, now you can scroll on flows using arrows
 
 
 On the very top you can see the ASN, the GEO location, and the virustotal score of each IP if available
 
-Check how to setup virustotal in Slips here https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#popup-notifications
+Check how to setup virustotal in Slips [here](https://stratospherelinuxips.readthedocs.io/en/develop/usage.html#popup-notifications).
 
 
 ### The Web Interface