chore(ua): remove default user agent spoof

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "website_crawler"
-version = "0.8.7"
+version = "0.8.8"
 authors = ["Jeff Mendez <[email protected]>"]
 edition = "2021"
 description = "gRPC tokio based web crawler"
@@ -23,7 +23,7 @@ regex = { version = "^1.5.0", optional = true }
 hashbrown = { version = "0.13.2" }
 log = "0.4.16"
 lazy_static = "1.4.0"
-ua_generator = { git = "https://github.com/a11ywatch/ua_generator.git", version = "0.3.5" }
+ua_generator = { git = "https://github.com/a11ywatch/ua_generator.git", version = "0.3.5", optional = true }
 percent-encoding = "2.1.0"
 env_logger = "0.9.0"
 string_concat = "0.0.1"
@@ -61,3 +61,4 @@ os_info = "3"
 [features]
 jemalloc = ["jemallocator", "jemalloc-sys"]
 regex = ["dep:regex"]
+ua_generator = ["dep:ua_generator"]

README.md

@@ -147,6 +147,7 @@ brew install protobuf
 
 `jemalloc` - use jemalloc memory allocator (default disabled).
 `regex` - use the regex crate for blacklist urls validation.
+`ua_generator` - use the ua_generator crate to spoof random user agent.
 
 ## About
 

src/lib.rs

@@ -1,6 +1,8 @@
 extern crate sitemap;
 extern crate tokio;
 extern crate tonic;
+
+#[cfg(feature = "ua_generator")]
 extern crate ua_generator;
 
 // packages mainly for spider

src/packages/spider/configuration.rs

@@ -44,3 +44,22 @@ impl Configuration {
         }
     }
 }
+
+/// get the user agent from the top agent list randomly.
+#[cfg(any(feature = "ua_generator"))]
+pub fn get_ua() -> &'static str {
+    ua_generator::ua::spoof_ua()
+}
+
+/// get the user agent via cargo package + version.
+#[cfg(not(any(feature = "ua_generator")))]
+pub fn get_ua() -> &'static str {
+    use std::env;
+
+    lazy_static! {
+        static ref AGENT: &'static str =
+            concat!(env!("CARGO_PKG_NAME"), '/', env!("CARGO_PKG_VERSION"));
+    };
+
+    AGENT.as_ref()
+}

src/packages/spider/website.rs

@@ -1,5 +1,5 @@
 use super::black_list::contains;
-use super::configuration::Configuration;
+use super::configuration::{Configuration, get_ua};
 use super::page::{build, get_page_selectors, Page};
 use super::robotparser::RobotFileParser;
 use super::utils::log;
@@ -28,7 +28,6 @@ use tokio::task;
 use tokio::task::JoinSet;
 use tokio_stream::StreamExt;
 use tonic::transport::Channel;
-use ua_generator::ua::spoof_ua;
 use url::Url;
 
 /// Represents a website to crawl and gather all links.
@@ -194,7 +193,7 @@ impl Website {
             .pool_idle_timeout(None)
             .user_agent(match &self.configuration.user_agent {
                 Some(ua) => ua.as_str(),
-                _ => spoof_ua(),
+                _ => &get_ua(),
             })
             .brotli(true);