Skip to content

Commit

Permalink
Merge pull request #512 from Ivanbeethoven/main
Browse files Browse the repository at this point in the history 
[scorpio]: mix Readonly dictionary FUSE and Overlay FS.
  • Loading branch information
@genedna
genedna authored Aug 15, 2024
2 parents da9dcad + 0a50c76 commit ec35f7c
Show file tree
Hide file tree
Showing 11 changed files with 516 additions and 128 deletions.
52 changes: 26 additions & 26 deletions docs/images/scorpio.drawio
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<mxfile host="65bd71144e">
<diagram id="05KLlnSNJxX6K2rn3go-" name="Page-1">
<mxGraphModel dx="1712" dy="517" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="583" pageHeight="827" math="1" shadow="0">
<mxGraphModel dx="1689" dy="530" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="583" pageHeight="827" math="1" shadow="0">
<root>
<mxCell id="0"/>
<mxCell id="1" parent="0"/>
Expand Down Expand Up @@ -76,84 +76,84 @@
<mxCell id="54" value="For ./c/" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="-351" y="528" width="60" height="30" as="geometry"/>
</mxCell>
<mxCell id="58" value="Inode-alloctor" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="58" value="Inode-alloctor" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="-330" y="250" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="63" style="edgeStyle=none;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="59" target="41">
<mxCell id="63" style="edgeStyle=none;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="59" target="41" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="65" style="edgeStyle=none;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="59" target="64">
<mxCell id="65" style="edgeStyle=none;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;" parent="1" source="59" target="64" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="59" value="mono dictionary" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="59" value="mono dictionary" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="-150" y="408" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="61" style="edgeStyle=none;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="60" target="59">
<mxCell id="61" style="edgeStyle=none;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="60" target="59" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="60" value="FUSE-request" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="60" value="FUSE-request" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="10" y="408" width="110" height="60" as="geometry"/>
</mxCell>
<mxCell id="64" value="Mega tree API" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="64" value="Mega tree API" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="-150" y="120" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="71" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="66" target="67">
<mxCell id="71" style="edgeStyle=none;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="66" target="67" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="72" style="edgeStyle=none;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="66" target="68">
<mxCell id="72" style="edgeStyle=none;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="66" target="68" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="73" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="66" target="69">
<mxCell id="73" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="66" target="69" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="66" value="FUSE" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="66" value="Mega&lt;br&gt;FUSE" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="230" y="160" width="110" height="50" as="geometry"/>
</mxCell>
<mxCell id="67" value="TREE read-only" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="67" value="DICfuse" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="110" y="260" width="110" height="60" as="geometry"/>
</mxCell>
<mxCell id="68" value="Overlay" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="68" value="Overlay" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="260" width="110" height="60" as="geometry"/>
</mxCell>
<mxCell id="69" value="Overlay" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="69" value="Overlay" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="370" y="260" width="110" height="60" as="geometry"/>
</mxCell>
<mxCell id="70" value="" style="shape=flexArrow;endArrow=classic;html=1;" edge="1" parent="1">
<mxCell id="70" value="" style="shape=flexArrow;endArrow=classic;html=1;" parent="1" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="280" y="70" as="sourcePoint"/>
<mxPoint x="280" y="150" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="81" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="74" target="68">
<mxCell id="81" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="74" target="68" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="74" value="passthrouth&amp;nbsp;" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="74" value="passthrouth&amp;nbsp;" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="354" width="110" height="60" as="geometry"/>
</mxCell>
<mxCell id="75" value="inode" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
<mxCell id="75" value="inode" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" parent="1" vertex="1">
<mxGeometry x="295" y="215" width="50" height="30" as="geometry"/>
</mxCell>
<mxCell id="85" style="edgeStyle=none;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="77" target="80">
<mxCell id="85" style="edgeStyle=none;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="77" target="80" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="77" value="readonly-文件层" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="77" value="readonly-文件层" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="380" y="442.64" width="110" height="62" as="geometry"/>
</mxCell>
<mxCell id="79" value="" style="edgeStyle=none;html=1;" edge="1" parent="1" source="78" target="74">
<mxCell id="79" value="" style="edgeStyle=none;html=1;" parent="1" source="78" target="74" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="78" value="write-文件层" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="78" value="write-文件层" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="380" y="354" width="110" height="60" as="geometry"/>
</mxCell>
<mxCell id="83" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="1" source="80">
<mxCell id="83" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" parent="1" source="80" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="295" y="450" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="84" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="80" target="74">
<mxCell id="84" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="80" target="74" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="80" value="passthrouth&amp;nbsp;" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxCell id="80" value="passthrouth&amp;nbsp;" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="442.64" width="110" height="60" as="geometry"/>
</mxCell>
</root>
Expand Down
2 changes: 2 additions & 0 deletions scorpio/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ sea-orm = { version= "0.12.15",features = [
] }
once_cell = "1.19.0"
tokio = { version = "1.38.1", features = ["full"] }
arc-swap = "1.5"
vm-memory = { version = "0.10", features = ["backend-mmap", "backend-bitmap"] }

[features]
async-io = []
Expand Down
2 changes: 2 additions & 0 deletions scorpio/init.bash
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
umount /tmp/dictest

4 changes: 3 additions & 1 deletion scorpio/src/dicfuse/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use tokio::task::JoinHandle;

use store::{DictionaryStore, IntoEntry};

struct Dicfuse{
pub struct Dicfuse{
store: Arc<DictionaryStore>,
//runtime: Arc<tokio::runtime::Runtime>,
}
Expand Down Expand Up @@ -41,6 +41,7 @@ impl FileSystem for Dicfuse{
type Handle = u64;

fn init(&self, capable:FsOptions) -> Result<FsOptions> {
println!("Dicfuse init....");
self.store.import();
//let mut ops = FsOptions::DO_READDIRPLUS | FsOptions::READDIRPLUS_AUTO;
Ok(fuse_backend_rs::abi::fuse_abi::FsOptions::empty())
Expand Down Expand Up @@ -389,6 +390,7 @@ mod tests {
});
// Wait for termination signal
let mut signals = Signals::new(TERM_SIGNALS).unwrap();
println!("Signals start");
if let Some(_sig) = signals.forever().next() {
//pass
}
Expand Down
10 changes: 6 additions & 4 deletions scorpio/src/dicfuse/store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,16 @@ use once_cell::sync::Lazy;
use radix_trie::{self, TrieCommon};
use std::sync::{Arc,Mutex};
use fuse_backend_rs::api::filesystem::DirEntry;
use crate::fuse::READONLY_INODE;

use super::fuse::{self, default_dic_entry, default_file_entry};
use super::model::GPath;
const MEGA_TREE_URL: &str = "localhost:8000";//TODO: make it configable
const UNKNOW_INODE: u64 = 0; // illegal inode number;
const INODE_FILE :&str ="file";
const INODE_DICTIONARY :&str ="directory";


#[derive(Serialize, Deserialize, Debug)]
pub struct Item {
name: String,
Expand Down Expand Up @@ -132,7 +136,6 @@ pub struct DictionaryStore {
radix_trie: Arc<Mutex<radix_trie::Trie<String, u64>>>,
}


#[allow(unused)]
impl DictionaryStore {
pub fn new() -> Self {
Expand All @@ -154,8 +157,10 @@ impl DictionaryStore {
}
fn update_inode(&self,pitem:Arc<DicItem>,item:Item){
self.next_inode.fetch_add(1, std::sync::atomic::Ordering::Relaxed);

let alloc_inode = self.next_inode.load(std::sync::atomic::Ordering::Relaxed);

assert!(alloc_inode < READONLY_INODE );
if item.content_type=="directory"{
self.queue.lock().unwrap().push_back(alloc_inode);
}
Expand All @@ -170,7 +175,6 @@ impl DictionaryStore {

}
pub fn import(&self){

let items: Vec<Item> = tokio::runtime::Runtime::new().unwrap().block_on(fetch_tree("")).unwrap().collect();//todo: can't tokio
let root_inode = self.inodes.lock().unwrap().get(&1).unwrap().clone();
for it in items{
Expand Down Expand Up @@ -205,7 +209,6 @@ impl DictionaryStore {
}
//queue.clear();
}


pub fn find_path(&self,inode :u64)-> Option<GPath>{
self.inodes.lock().unwrap().get(&inode).map(|item| item.path_name.clone())
Expand Down Expand Up @@ -233,7 +236,6 @@ impl DictionaryStore {
todo!()
}


pub fn do_readdir(&self,
ctx: &Context,
inode: u64,
Expand Down
10 changes: 5 additions & 5 deletions scorpio/src/fuse/inode_alloc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ use std::{collections::HashMap, sync::{atomic::{AtomicU64, Ordering}, Mutex}};

#[allow(unused)]
const VFS_MAX_INO: u64 = 0xff_ffff_ffff_ffff;
#[allow(unused)]
const READONLY_INODE :u64 = 0xffff_ffff;
pub const READONLY_INODE :u64 = 0xffff_ffff;
// Alloc inode numbers at one batch
#[allow(unused)]
const INODE_ALLOC_BATCH:u64 = 0x1_0000_0000;
Expand All @@ -22,14 +21,15 @@ impl InodeAlloc{
alloc: Mutex::new(HashMap::new())
}
}
pub fn alloc_inode(&mut self,ovl_inode:u64)-> u64{
pub fn alloc_inode(&self,ovl_inode:u64)-> u64{
self.next_ino_batch.fetch_add(1, Ordering::Relaxed);
let ainode = self.next_ino_batch.load(Ordering::Acquire);
let mut alloc = self.alloc.lock().unwrap();
alloc.insert(ainode,ovl_inode);
ainode
}
pub fn get_ovl_inode(&self,path_inode:u64)-> Option<u64>{
self.alloc.lock().unwrap().get(&path_inode).copied()
pub fn get_ovl_inode(&self,inode_batch:u64)-> Option<u64>{
self.alloc.lock().unwrap().get(&inode_batch).copied()
}

}
Loading

1 comment on commit ec35f7c

@vercel
Copy link

@vercel vercel bot commented on ec35f7c Aug 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

mega – ./

mega-gitmono.vercel.app
www.gitmega.dev
gitmega.dev
mega-git-main-gitmono.vercel.app

Please sign in to comment.