Merge pull request keep-starknet-strange#4 from karnotxyz/orchestrato…

…r_refactor

refactor to workspace

Cargo.toml

@@ -1,39 +1,38 @@
-[package]
-name = "madara-orchestrator"
+[workspace]
+resolver = "2"
+members = [
+  "crates/orchestrator",
+  "crates/da_clients/da-client-interface",
+  "crates/da_clients/ethereum",
+  "crates/utils",
+]
+
+[workspace.package]
 version = "0.1.0"
 edition = "2021"
 authors = ["Apoorv Sadana <@apoorvsadana>"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[dependencies]
-alloy = { git = "https://github.com/alloy-rs/alloy", rev = "86027c9bb984f3a12a30ffd2a3c5f2f06595f1d6", features = [
-  "providers",
-  "rpc-client",
-  "transport-http",
-], optional = true }
-async-trait = "0.1.77"
-axum = { version = "0.7.4", features = ["macros"] }
-axum-macros = "0.4.1"
-color-eyre = "0.6.2"
-dotenvy = "0.15.7"
-futures = "0.3.30"
-mongodb = { version = "2.8.1", features = ["bson-uuid-1"], optional = true }
-omniqueue = { version = "0.2.0", optional = true }
-reqwest = { version = "0.11.24", optional = true }
-rstest = "0.18.2"
+[workspace.dependencies]
+async-trait = { version = "0.1.77" }
+axum = { version = "0.7.4" }
+axum-macros = { version = "0.4.1" }
+color-eyre = { version = "0.6.2" }
+dotenvy = { version = "0.15.7" }
+futures = { version = "0.3.30" }
+mongodb = { version = "2.8.1" }
+omniqueue = { version = "0.2.0" }
+rstest = { version = "0.18.2" }
 serde = { version = "1.0.197" }
-serde_json = "1.0.114"
-starknet = "0.9.0"
-thiserror = "1.0.57"
-tokio = { version = "1.36.0", features = ["sync", "macros", "rt-multi-thread"] }
-tracing = "0.1.40"
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
-url = "2.5.0"
-uuid = { version = "1.7.0", features = ["v4", "serde"] }
-
-[features]
-default = ["ethereum", "with_mongdb", "with_sqs"]
-ethereum = ["alloy", "reqwest"]
-with_mongdb = ["mongodb"]
-with_sqs = ["omniqueue"]
+serde_json = { version = "1.0.114" }
+starknet = { version = "0.9.0" }
+thiserror = { version = "1.0.57" }
+tokio = { version = "1.36.0" }
+tracing = { version = "0.1.40" }
+tracing-subscriber = { version = "0.3.18" }
+url = { version = "2.5.0" }
+uuid = { version = "1.7.0" }
+da-client-interface = { path = "crates/da_clients/da-client-interface" }
+ethereum-da-client = { path = "crates/da_clients/ethereum" }
+utils = { path = "crates/utils" }

crates/da_clients/da-client-interface/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "da-client-interface"
+version.workspace = true
+edition.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = { workspace = true }
+axum = { workspace = true }
+color-eyre = { workspace = true }
+starknet = { workspace = true }

src/da_clients/mod.rs → crates/da_clients/da-client-interface/src/lib.rs

@@ -1,10 +1,16 @@
-use crate::jobs::types::JobVerificationStatus;
-use axum::async_trait;
+use async_trait::async_trait;
 use color_eyre::Result;
 use starknet::core::types::FieldElement;
 
-/// Ethereum client
-pub mod ethereum;
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum DaVerificationStatus {
+    #[allow(dead_code)]
+    Pending,
+    #[allow(dead_code)]
+    Verified,
+    #[allow(dead_code)]
+    Rejected,
+}
 
 /// Trait for every new DaClient to implement
 #[async_trait]
@@ -13,7 +19,7 @@ pub trait DaClient: Send + Sync {
     /// which can be used to track the status of the DA transaction.
     async fn publish_state_diff(&self, state_diff: Vec<FieldElement>) -> Result<String>;
     /// Should verify the inclusion of the state diff in the DA layer and return the status
-    async fn verify_inclusion(&self, external_id: &str) -> Result<JobVerificationStatus>;
+    async fn verify_inclusion(&self, external_id: &str) -> Result<DaVerificationStatus>;
 }
 
 /// Trait for every new DaConfig to implement

crates/da_clients/ethereum/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "ethereum-da-client"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+alloy = { git = "https://github.com/alloy-rs/alloy", rev = "86027c9bb984f3a12a30ffd2a3c5f2f06595f1d6", features = [
+  "providers",
+  "rpc-client",
+  "transport-http",
+] }
+async-trait = { workspace = true }
+color-eyre = { workspace = true }
+da-client-interface = { workspace = true }
+reqwest = { version = "0.11.24" }
+starknet = { workspace = true }
+url = { workspace = true }
+utils = { workspace = true }

src/da_clients/ethereum/config.rs → crates/da_clients/ethereum/src/config.rs

@@ -1,5 +1,5 @@
-use crate::da_clients::DaConfig;
-use crate::utils::env_utils::get_env_var_or_panic;
+use da_client_interface::DaConfig;
+use utils::env_utils::get_env_var_or_panic;
 
 #[derive(Clone, Debug)]
 pub struct EthereumDaConfig {

src/da_clients/ethereum/mod.rs → crates/da_clients/ethereum/src/lib.rs

@@ -9,9 +9,8 @@ use starknet::core::types::FieldElement;
 use std::str::FromStr;
 use url::Url;
 
-use crate::da_clients::ethereum::config::EthereumDaConfig;
-use crate::da_clients::DaClient;
-use crate::jobs::types::JobVerificationStatus;
+use config::EthereumDaConfig;
+use da_client_interface::{DaClient, DaVerificationStatus};
 
 pub mod config;
 pub struct EthereumDaClient {
@@ -25,7 +24,7 @@ impl DaClient for EthereumDaClient {
         unimplemented!()
     }
 
-    async fn verify_inclusion(&self, _external_id: &str) -> Result<JobVerificationStatus> {
+    async fn verify_inclusion(&self, _external_id: &str) -> Result<DaVerificationStatus> {
         todo!()
     }
 }

crates/orchestrator/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "orchestrator"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+async-trait = { workspace = true }
+axum = { workspace = true, features = ["macros"] }
+axum-macros = { workspace = true }
+color-eyre = { workspace = true }
+da-client-interface = { workspace = true }
+dotenvy = { workspace = true }
+ethereum-da-client = { workspace = true, optional = true }
+futures = { workspace = true }
+mongodb = { workspace = true, features = ["bson-uuid-1"], optional = true }
+omniqueue = { workspace = true, optional = true }
+rstest = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+starknet = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+url = { workspace = true }
+uuid = { workspace = true, features = ["v4", "serde"] }
+
+[features]
+default = ["ethereum", "with_mongodb", "with_sqs"]
+ethereum = ["ethereum-da-client"]
+with_mongodb = ["mongodb"]
+with_sqs = ["omniqueue"]

src/config.rs → crates/orchestrator/src/config.rs

@@ -1,13 +1,13 @@
-use crate::da_clients::ethereum::config::EthereumDaConfig;
-use crate::da_clients::ethereum::EthereumDaClient;
-use crate::da_clients::{DaClient, DaConfig};
 use crate::database::mongodb::config::MongoDbConfig;
 use crate::database::mongodb::MongoDb;
 use crate::database::{Database, DatabaseConfig};
 use crate::queue::sqs::SqsQueue;
 use crate::queue::QueueProvider;
 use crate::utils::env_utils::get_env_var_or_panic;
+use da_client_interface::{DaClient, DaConfig};
 use dotenvy::dotenv;
+use ethereum_da_client::config::EthereumDaConfig;
+use ethereum_da_client::EthereumDaClient;
 use starknet::providers::jsonrpc::HttpTransport;
 use starknet::providers::{JsonRpcClient, Url};
 use std::sync::Arc;

src/jobs/da_job/mod.rs → crates/orchestrator/src/jobs/da_job/mod.rs

@@ -49,7 +49,7 @@ impl Job for DaJob {
     }
 
     async fn verify_job(&self, config: &Config, job: &JobItem) -> Result<JobVerificationStatus> {
-        Ok(config.da_client().verify_inclusion(job.external_id.unwrap_string()?).await?)
+        Ok(config.da_client().verify_inclusion(job.external_id.unwrap_string()?).await?.into())
     }
 
     fn max_process_attempts(&self) -> u64 {

src/jobs/types.rs → crates/orchestrator/src/jobs/types.rs

@@ -1,4 +1,6 @@
 use color_eyre::{eyre::eyre, Result};
+use da_client_interface::DaVerificationStatus;
+// TODO: job types shouldn't depend on mongodb
 use mongodb::bson::serde_helpers::uuid_1_as_binary;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -123,3 +125,13 @@ pub enum JobVerificationStatus {
     #[allow(dead_code)]
     Rejected,
 }
+
+impl From<DaVerificationStatus> for JobVerificationStatus {
+    fn from(status: DaVerificationStatus) -> Self {
+        match status {
+            DaVerificationStatus::Pending => JobVerificationStatus::Pending,
+            DaVerificationStatus::Verified => JobVerificationStatus::Verified,
+            DaVerificationStatus::Rejected => JobVerificationStatus::Rejected,
+        }
+    }
+}

src/main.rs → crates/orchestrator/src/main.rs

@@ -2,8 +2,6 @@
 mod config;
 /// Controllers for the routes
 mod controllers;
-/// Contains the trait that all DA clients must implement
-mod da_clients;
 /// Contains the trait that all database clients must implement
 mod database;
 /// Contains the trait that all jobs must implement. Also

crates/utils/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "utils"
+version.workspace = true
+edition.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+color-eyre = { workspace = true }

crates/utils/src/env_utils.rs

@@ -0,0 +1,13 @@
+use color_eyre::Result;
+
+pub fn get_env_var(key: &str) -> Result<String> {
+    std::env::var(key).map_err(|e| e.into())
+}
+
+pub fn get_env_var_or_panic(key: &str) -> String {
+    get_env_var(key).unwrap_or_else(|e| panic!("Failed to get env var {}: {}", key, e))
+}
+
+pub fn get_env_var_or_default(key: &str, default: &str) -> String {
+    get_env_var(key).unwrap_or(default.to_string())
+}

crates/utils/src/lib.rs

@@ -0,0 +1 @@
+pub mod env_utils;