Chunkify withdrawals records (#152)

* Chunkify withdrawals records Signed-off-by: cyc60 <[email protected]> * Fix chunk call Signed-off-by: cyc60 <[email protected]> * Version bump Signed-off-by: cyc60 <[email protected]> * Move chunk size to settings Signed-off-by: cyc60 <[email protected]> --------- Signed-off-by: cyc60 <[email protected]>
stakewise · Aug 24, 2023 · 8a61f8f · 8a61f8f
1 parent b315e68
commit 8a61f8f
Show file tree

Hide file tree

Showing 4 changed files with 417 additions and 395 deletions.
diff --git a/oracle/oracle/rewards/controller.py b/oracle/oracle/rewards/controller.py
@@ -25,6 +25,7 @@
     MGNO_RATE,
     NETWORK,
     NETWORK_CONFIG,
+    ORACLE_WITHDRAWAL_CHUNK_SIZE,
     REWARD_VOTE_FILENAME,
     WAD,
 )
@@ -221,6 +222,33 @@ async def calculate_withdrawal_rewards(
         )
         execution_client = get_web3_client()
 
+        chunk_size = ORACLE_WITHDRAWAL_CHUNK_SIZE
+        for block_number in range(from_block, to_block, chunk_size):
+            withdrawals_amount += await self.fetch_withdrawal_chunk(
+                validator_indexes=validator_indexes,
+                from_block=block_number,
+                to_block=min(block_number + chunk_size, to_block),
+                execution_client=execution_client,
+            )
+
+        withdrawals_amount = Web3.toWei(withdrawals_amount, "gwei")
+        if NETWORK == GNOSIS_CHAIN:
+            # apply mGNO <-> GNO exchange rate
+            withdrawals_amount = Wei(int(withdrawals_amount * WAD // MGNO_RATE))
+        return withdrawals_amount
+
+    async def fetch_withdrawal_chunk(
+        self,
+        validator_indexes: set[int],
+        from_block: int,
+        to_block: int,
+        execution_client,
+    ) -> int:
+        logger.info(
+            f"Retrieving pool validator withdrawals chunk "
+            f"from block: {from_block} to block: {to_block}"
+        )
+        withdrawals_amount = 0
         with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
             futures = [
                 executor.submit(get_withdrawals, execution_client, block_number)
@@ -231,11 +259,6 @@ async def calculate_withdrawal_rewards(
                 for withdrawal in withdrawals:
                     if withdrawal["validator_index"] in validator_indexes:
                         withdrawals_amount += withdrawal["amount"]
-
-        withdrawals_amount = Web3.toWei(withdrawals_amount, "gwei")
-        if NETWORK == GNOSIS_CHAIN:
-            # apply mGNO <-> GNO exchange rate
-            withdrawals_amount = Wei(int(withdrawals_amount * WAD // MGNO_RATE))
         return withdrawals_amount
 
     async def get_withdrawals_from_block(self, current_slot: int) -> BlockNumber | None:

diff --git a/oracle/settings.py b/oracle/settings.py
@@ -28,6 +28,10 @@
 
 # oracle
 ORACLE_PROCESS_INTERVAL = config("ORACLE_PROCESS_INTERVAL", default=15, cast=int)
+ORACLE_WITHDRAWAL_CHUNK_SIZE = config(
+    "ORACLE_WITHDRAWAL_CHUNK_SIZE", default=50000, cast=int
+)
+
 
 IPFS_FETCH_ENDPOINTS = config(
     "IPFS_FETCH_ENDPOINTS",