Merge remote-tracking branch 'origin/main' into mk/script

.gitignore

@@ -24,5 +24,10 @@ Cargo.lock
 .venv
 .python-version
 __pycache__
+.ipynb_checkpoints/
+*.ipynb
+
+.client_cache/
+.utxo_data/
+.timestamps_data/
 
-.client_cache/

README.md

@@ -11,7 +11,7 @@
 
 ## Overview
 
-Raito is a zero-knowledge Bitcoin client implemented in Cairo. It aims to provide trustless validation of the Bitcoin blockchain through STARK proof verification. It is heavily inspired by [ZeroSync](https://github.com/ZeroSync/ZeroSync) project.
+Shinigami is a zero-knowledge Bitcoin client implemented in Cairo. It aims to provide trustless validation of the Bitcoin blockchain through STARK proof verification. It is heavily inspired by [ZeroSync](https://github.com/ZeroSync/ZeroSync) project.
 
 > **Disclaimer:** This project is in the early stages of development and should not be used in production. It will evolve rapidly, expect breaking changes.
 
@@ -36,7 +36,7 @@ style Vp fill:gold
 style Vb fill:gold
 ```
 
-Raito, at its core, accepts two inputs: a batch of consecutive blocks <i>n</i> to <i>m</i> and a STARK proof of the state of the chain up to block <i>n−1</i>. It ensures that the historical chain state is valid by verifying the STARK proof. Then, it produces a new chain state by applying the new blocks on top of the historical state. As a result, a proof of the new state is generated.
+At its core, consensus client accepts two inputs: a batch of consecutive blocks <i>n</i> to <i>m</i> and a STARK proof of the state of the chain up to block <i>n−1</i>. It ensures that the historical chain state is valid by verifying the STARK proof. Then, it produces a new chain state by applying the new blocks on top of the historical state. As a result, a proof of the new state is generated.
 
 ## Applications
 
@@ -92,7 +92,7 @@ Try to run script validation with external Cairo crate.
 
 Tasks:
 
-* [ ] Integrate Shinigami
+* [ ] Integrate Shinigami-script
 
 ### Milestone 4 - UTXO set verification
 
@@ -114,24 +114,21 @@ Tasks:
 
 Validate full block execution, including the Bitcoin scripts checks and Utreexo.
 
-### Milestone 6 - Proving the validation
+### Milestone 6 - Optimizations
 
-Recursively verify STARK proofs of chain state updates.
+* [ ] identify Cairo code botlenecks
+* [ ] consider using garaga msm to batch signature verifications
 
-## Name reference
+### Milestone 7 - Proving the validation
 
-Raito is a reference to Light Yagami (夜神月, Yagami Raito) from the manga/anime Death Note.
+Recursively verify STARK proofs of chain state updates. Still largely tbd.
 
-* Raito in Japanese means "Light", which in turns can refer to Lightning ⚡ (and hence both a reference to speed of verification of the Bitcoin blockchain using a ZKP and a reference to the Lightning Network)
-* Raito can work in tandem with [Shinigami](https://github.com/keep-starknet-strange/shinigami) that enables verification of Bitcoin Script programs. Raito = Consensus and Shinigami = Execution. Since Shinigami was named after Ryuk (Shinigami in Death Note), Raito was named after Light (Raito in Death Note).
-* What Raito writes in the Death Note always happen, so you can see it as a source of truth, similarly to how you use a Zero-Knowledge Proof to verify the integrity of a computation.
-
-![Raito and Raito](./docs/img/memes/raito_shinigami_fusion.jpg)
+* [ ] use garaga to implement signature verifications
 
 # Contact
 
-* [Raito Telegram](https://t.me/ShinigamiStarknet)
-* [Raito OnlyDust](https://app.onlydust.com/p/raito---bitcoin-zk-client)
+* [Telegram](https://t.me/ShinigamiStarknet)
+* [OnlyDust](https://app.onlydust.com/p/raito---bitcoin-zk-client)
 
 ## Usage
 

Scarb.lock

@@ -6,6 +6,7 @@ name = "client"
 version = "0.1.0"
 dependencies = [
  "consensus",
+ "utreexo",
 ]
 
 [[package]]
@@ -63,3 +64,7 @@ source = "git+https://github.com/keep-starknet-strange/shinigami.git?rev=72ed4f9
 [[package]]
 name = "utils"
 version = "0.1.0"
+
+[[package]]
+name = "utreexo"
+version = "0.1.0"

packages/client/README.md

@@ -33,6 +33,12 @@ Run a specific test file (or several files):
 scarb test tests/data/light_481823.json
 ```
 
+Show output for a specific test:
+
+```sh
+scarb test tests/data/light_481823.json --nocapture
+```
+
 Re-generate integration test data:
 
 ```sh

packages/client/Scarb.toml

@@ -5,6 +5,7 @@ edition = "2024_07"
 
 [dependencies]
 consensus = { path = "../consensus" }
+utreexo = { path = "../utreexo" }
 
 [dev-dependencies]
 cairo_test.workspace = true
@@ -13,4 +14,4 @@ cairo_test.workspace = true
 test = "scarb build && ../../scripts/data/integration_tests.sh"
 regenerate_tests= "../../scripts/data/regenerate_tests.sh"
 client = "scarb build && ../../scripts/data/client.sh"
-lint = "flake8 scripts/ && black --check scripts/ && scarb fmt"
+lint = "black ../../scripts/data && flake8 ../../scripts/data --max-complexity=10 --max-line-length=88 && scarb fmt"

packages/client/src/main.cairo

@@ -1,14 +1,12 @@
 use consensus::types::block::Block;
-use consensus::types::state::State;
-use consensus::types::chain_state::BlockValidator;
-use consensus::types::utreexo::UtreexoStateTrait;
-use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait};
+use consensus::types::chain_state::{ChainState, BlockValidator};
+use consensus::types::utxo_set::UtxoSet;
 
 /// Raito program arguments.
 #[derive(Serde)]
 struct Args {
-    /// Current (initial) state
-    state: State,
+    /// Current (initial) chain state
+    chain_state: ChainState,
     /// Batch of blocks that have to be applied to the current chain state
     blocks: Array<Block>,
 }
@@ -18,26 +16,18 @@ struct Args {
 /// Receives current state (chain state + utreexo state) and pending blocks,
 /// then validates and applies them one by one.
 /// Returns new state in case of success, otherwise raises an error.
-fn main(mut arguments: Span<felt252>) -> State {
-    let Args { mut state, blocks, } = Serde::deserialize(ref arguments)
+fn main(mut arguments: Span<felt252>) -> ChainState {
+    let Args { mut chain_state, blocks, } = Serde::deserialize(ref arguments)
         .expect('Failed to deserialize');
 
     let mut utxo_set: UtxoSet = Default::default();
 
     // Validate and apply block, accumulating UTXO updates in utxo_set
     for block in blocks {
-        state
-            .chain_state = state
-            .chain_state
+        chain_state = chain_state
             .validate_and_apply(block, ref utxo_set)
             .expect('Block validation failed');
     };
 
-    // Validate and apply UTXO updates
-    state.utreexo_state.validate_and_apply(ref utxo_set).expect('Utreexo validation failed');
-
-    // Ensure all UTXOs have been processed
-    utxo_set.finalize().expect('UtxoSet finalization failed');
-
-    state
+    chain_state
 }

packages/client/src/test.cairo

@@ -1,18 +1,33 @@
 use consensus::types::block::Block;
 use consensus::types::chain_state::{ChainState, BlockValidatorImpl};
-use consensus::types::state::{State};
 use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait};
+use utreexo::vanilla::state::{UtreexoState, UtreexoStateTrait};
+use utreexo::vanilla::proof::UtreexoProof;
 use core::testing::get_available_gas;
+use core::serde::Serde;
 
 /// Integration testing program arguments.
-#[derive(Serde)]
+#[derive(Drop)]
 struct Args {
     /// Current (initial) chain state
     chain_state: ChainState,
     /// Batch of blocks that have to be applied to the current chain state
     blocks: Array<Block>,
     /// Expected chain state (that we want to compare the result with)
     expected_chain_state: ChainState,
+    /// Optional Utreexo arguments
+    utreexo_args: Option<UtreexoArgs>,
+}
+
+/// Utreexo arguments necessary for constraining the UTXO set
+#[derive(Drop, Serde)]
+struct UtreexoArgs {
+    /// Current (initial) accumulator state
+    state: UtreexoState,
+    /// Inclusion proofs for TXOs spent during program run
+    proofs: Array<UtreexoProof>,
+    /// Expected accumulator state at the end of the execution
+    expected_state: UtreexoState,
 }
 
 /// Integration testing program entrypoint.
@@ -21,21 +36,19 @@ struct Args {
 /// Panics in case of a validation error or chain state mismatch.
 /// Prints result to the stdout.
 fn test(mut arguments: Span<felt252>) {
-    let Args { mut chain_state, blocks, expected_chain_state } = Serde::deserialize(ref arguments)
+    let Args { mut chain_state, blocks, expected_chain_state, utreexo_args } = Serde::deserialize(
+        ref arguments
+    )
         .expect('Failed to deserialize');
 
-    // Temporary solution while script doesn't handle utreexo.
-    // Allows to test one isolated block, or a batch of blocks starting from genesis.
-    let mut state: State = State { chain_state: chain_state, utreexo_state: Default::default(), };
     let mut utxo_set: UtxoSet = Default::default();
-
     let mut gas_before = get_available_gas();
 
     for block in blocks {
-        let height = state.chain_state.block_height + 1;
-        match state.chain_state.validate_and_apply(block, ref utxo_set) {
+        let height = chain_state.block_height + 1;
+        match chain_state.validate_and_apply(block, ref utxo_set) {
             Result::Ok(new_chain_state) => {
-                state.chain_state = new_chain_state;
+                chain_state = new_chain_state;
                 let gas_after = get_available_gas();
                 println!("OK: block={} gas_spent={}", height, gas_before - gas_after);
                 gas_before = gas_after;
@@ -50,43 +63,64 @@ fn test(mut arguments: Span<felt252>) {
         }
     };
 
-    if state.chain_state != expected_chain_state {
+    if chain_state != expected_chain_state {
         println!(
             "FAIL: block={} error='expected chain state {:?}, actual {:?}'",
-            state.chain_state.block_height,
+            chain_state.block_height,
             expected_chain_state,
-            state.chain_state
+            chain_state
         );
         panic!();
     }
 
-    // TODO: provide the expected utreexo state via args and compare it with the actual one
-    //
-    // gas_before = get_available_gas();
-    //
-    // match state.utreexo_state.validate_and_apply(ref utxo_set) {
-    //     Result::Ok(()) => {
-    //         let gas_after = get_available_gas();
-    //         println!("OK: gas_spent={}", gas_before - gas_after);
-    //     },
-    //     Result::Err(err) => {
-    //         let gas_after = get_available_gas();
-    //         println!("FAIL: gas_spent={} error='{:?}'", gas_before - gas_after, err);
-    //         panic!();
-    //     }
-    // }
-    //
-    // if state.utreexo_state != expected_utreexo_state {
-    //     println!(
-    //         "FAIL: error='expected utreexo state {:?}, actual {:?}'",
-    //         expected_utreexo_state,
-    //         state.utreexo_state
-    //     );
-    //     panic!();
-    // }
-
     if let Result::Err(err) = utxo_set.finalize() {
         println!("FAIL: error='{}'", err);
         panic!();
     }
+
+    if let Option::Some(UtreexoArgs { mut state, proofs, expected_state }) = utreexo_args {
+        match state
+            .validate_and_apply(
+                utxo_set.leaves_to_add.span(), utxo_set.leaves_to_delete.span(), proofs.span()
+            ) {
+            Result::Ok(new_state) => {
+                state = new_state;
+                let gas_after = get_available_gas();
+                println!("OK: gas_spent={}", gas_before - gas_after);
+            },
+            Result::Err(err) => {
+                let gas_after = get_available_gas();
+                println!("FAIL: gas_spent={} error='{:?}'", gas_before - gas_after, err);
+                panic!();
+            }
+        }
+
+        if state != expected_state {
+            println!(
+                "FAIL: error='expected utreexo state {:?}, actual {:?}'", expected_state, state
+            );
+            panic!();
+        }
+    }
+}
+
+/// Workaround for handling missing `utreexo_args` field.
+/// Rough analogue of `#[serde(default)]`
+impl ArgsSerde of Serde<Args> {
+    fn serialize(self: @Args, ref output: Array<felt252>) {
+        panic!("not implemented");
+    }
+
+    fn deserialize(ref serialized: Span<felt252>) -> Option<Args> {
+        let chain_state: ChainState = Serde::deserialize(ref serialized).expect('chain_state');
+        let blocks: Array<Block> = Serde::deserialize(ref serialized).expect('blocks');
+        let expected_chain_state: ChainState = Serde::deserialize(ref serialized)
+            .expect('expected_chain_state');
+        let utreexo_args: Option<UtreexoArgs> = if serialized.len() > 0 {
+            Option::Some(Serde::deserialize(ref serialized).expect('utreexo_args'))
+        } else {
+            Option::None
+        };
+        Option::Some(Args { chain_state, blocks, expected_chain_state, utreexo_args, })
+    }
 }