OCT-1355: Aggregate block rewards

backend/app/constants.py

@@ -12,3 +12,4 @@
 
 BEACONCHAIN_API = "https://beaconcha.in/api"
 ETHERSCAN_API = "https://api.etherscan.io/api"
+BITQUERY_API = "https://graphql.bitquery.io"

backend/app/context/epoch_details.py

@@ -3,6 +3,7 @@
 
 from app.context.epoch_state import EpochState
 from app.context.helpers import check_if_future
+from app.exceptions import WrongBlocksRange
 from app.extensions import epochs
 from app.infrastructure import graphql
 from app.infrastructure.external_api.etherscan.blocks import get_block_num_from_ts
@@ -44,6 +45,20 @@ def __init__(
             self.remaining_sec = remaining_sec
 
         self.remaining_days = sec_to_days(self.remaining_sec)
+        self.block_rewards = None
+
+    @property
+    def duration_range(self) -> tuple[int, int]:
+        return self.start_sec, self.end_sec
+
+    @property
+    def no_blocks(self):
+        """
+        Returns the number of blocks within [start_block, end_block) in the epoch.
+        """
+        if not self.end_block or not self.start_block:
+            raise WrongBlocksRange
+        return self.end_block - self.start_block
 
 
 def get_epoch_details(epoch_num: int, epoch_state: EpochState) -> EpochDetails:

backend/app/exceptions.py

@@ -238,3 +238,11 @@ class EmptyAllocations(OctantException):
 
     def __init__(self):
         super().__init__(self.description, self.code)
+
+
+class WrongBlocksRange(OctantException):
+    code = 400
+    description = "Attempt to use wrong range of start and end block in epoch"
+
+    def __init__(self):
+        super().__init__(self.description, self.code)

backend/app/infrastructure/external_api/bitquery/blocks_reward.py

@@ -0,0 +1,47 @@
+import requests
+
+
+import app as app_module
+from app.constants import BITQUERY_API
+from app.exceptions import ExternalApiException
+from app.infrastructure.external_api.bitquery.req_producer import (
+    produce_payload,
+    BitQueryActions,
+    get_bitquery_header,
+)
+
+
+def get_blocks_rewards(
+    address: str, start_time: str, end_time: str, limit: int
+) -> list:
+    """
+    Fetch Ethereum blocks within a specified time range in ascending order by timestamp.
+
+    Args:
+    - start_time (str): The start time in ISO 8601 format.
+    - end_time (str): The end time in ISO 8601 format.
+    - address (str): The miner (fee recipient) address.
+    - limit (int): The number of blocks to retrieve starting from start_time.
+                   Useful whilst getting end_blocks exclusively from epochs.
+    """
+    payload = produce_payload(
+        action_type=BitQueryActions.GET_BLOCK_REWARDS,
+        address=address,
+        start_time=start_time,
+        end_time=end_time,
+        limit=limit,
+    )
+    headers = get_bitquery_header()
+
+    api_url = BITQUERY_API
+
+    try:
+        response = requests.request("POST", api_url, headers=headers, data=payload)
+        response.raise_for_status()
+        json_response = response.json()
+    except requests.exceptions.RequestException as e:
+        app_module.ExceptionHandler.print_stacktrace(e)
+        raise ExternalApiException(api_url, e, 500)
+
+    blocks = json_response.json()["data"]["ethereum"]["blocks"]
+    return blocks

backend/app/infrastructure/external_api/bitquery/req_producer.py

@@ -0,0 +1,57 @@
+import json
+from enum import IntEnum
+
+from flask import current_app as app
+
+
+class BitQueryActions(IntEnum):
+    GET_BLOCK_REWARDS = 0
+
+
+def get_bitquery_header():
+    headers = {
+        "Content-Type": "application/json",
+        "X-API-KEY": app.config["BITQUERY_API_KEY"],
+        "Authorization": app.config["BITQUERY_BEARER"],
+    }
+
+    return headers
+
+
+def produce_payload(action_type: BitQueryActions, **query_values) -> str:
+    payloads_variations = {BitQueryActions.GET_BLOCK_REWARDS: _block_rewards_payload}
+
+    return payloads_variations[action_type](**query_values)
+
+
+def _block_rewards_payload(
+    start_time: str, end_time: str, address: str, limit: int, **kwargs
+) -> str:
+    payload = json.dumps(
+        {
+            "query": f"""query ($network: EthereumNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {{
+          ethereum(network: $network) {{
+            blocks(time: {{since: $from, till: $till}}, options: {{asc: "timestamp.unixtime", limit: {limit}}}) {{
+              timestamp {{
+                unixtime
+              }}
+              reward
+              address: miner(miner: {{is: "{address}"}}) {{
+                address
+              }}
+            }}
+          }}
+        }}""",
+            "variables": json.dumps(
+                {
+                    "network": "ethereum",
+                    "from": start_time,
+                    "till": end_time,
+                    "limit": limit,
+                    "dateFormat": "%Y-%m-%d",
+                }
+            ),
+        }
+    )
+
+    return payload

backend/app/infrastructure/external_api/etherscan/blocks.py

@@ -12,7 +12,7 @@
 
 def get_block_num_from_ts(timestamp: int) -> int:
     app.logger.debug(f"Getting block number from timestamp: {timestamp}")
-    api_url = _get_api_url(timestamp, BlockAction.BLOCK)
+    api_url = _get_api_url(timestamp, BlockAction.BLOCK_NO_BY_TS)
     try:
         response = requests.get(api_url)
         raise_for_status(response)

backend/app/infrastructure/external_api/etherscan/req_params.py

@@ -8,7 +8,8 @@ class AccountAction(StrEnum):
 
 
 class BlockAction(StrEnum):
-    BLOCK = "getblocknobytime"
+    BLOCK_NO_BY_TS = "getblocknobytime"
+    BLOCK_REWARD = "getblockreward"
 
 
 class ClosestValue(StrEnum):

backend/app/legacy/utils/time.py

@@ -19,6 +19,9 @@ def timestamp_s(self) -> float:
     def datetime(self) -> DateTime:
         return DateTime.fromtimestamp(self.timestamp_s())
 
+    def to_isoformat(self):
+        return self.datetime().isoformat()
+
     def __eq__(self, o):
         if isinstance(o, Timestamp):
             return self._timestamp_us == o._timestamp_us
@@ -60,3 +63,8 @@ def sec_to_days(sec: int) -> int:
 
 def days_to_sec(days: int) -> int:
     return int(days * 86400)
+
+
+def timestamp_to_isoformat(timestamp_sec: int) -> str:
+    timestamp = from_timestamp_s(timestamp_sec)
+    return timestamp.to_isoformat()

backend/app/modules/staking/proceeds/core.py

@@ -1,3 +1,5 @@
+from decimal import Decimal
+
 import pandas as pd
 from gmpy2 import mpz
 
@@ -54,8 +56,15 @@ def sum_withdrawals(withdrawals_txs: list[dict]) -> int:
     return w3.to_wei(int(total_gwei), "gwei")
 
 
-def aggregate_proceeds(mev: int, withdrawals: int) -> int:
-    return mev + withdrawals
+def sum_blocks_rewards(blocks_rewards: list) -> int:
+    df = pd.DataFrame(blocks_rewards)
+    blocks_reward_eth = df["reward"].apply(Decimal).sum()
+
+    return int(w3.to_wei(blocks_reward_eth, "ether"))
+
+
+def aggregate_proceeds(mev: int, withdrawals: int, blocks_rewards: list) -> int:
+    return mev + withdrawals + sum_blocks_rewards(blocks_rewards)
 
 
 def _filter_deposit_withdrawals(amount: mpz) -> mpz:

backend/app/modules/staking/proceeds/service/aggregated.py

@@ -5,6 +5,8 @@
     get_transactions,
     AccountAction,
 )
+from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_rewards
+from app.legacy.utils.time import timestamp_to_isoformat
 from app.modules.staking.proceeds.core import (
     sum_mev,
     sum_withdrawals,
@@ -14,6 +16,24 @@
 
 
 class AggregatedStakingProceeds(Model):
+    def _retrieve_blocks_rewards(
+        self, start_sec: int, end_sec: int, withdrawals_target: str, limit: int
+    ) -> list:
+        blocks_rewards = []
+
+        if end_sec is None:
+            return blocks_rewards
+
+        start_datetime, end_datetime = (
+            timestamp_to_isoformat(start_sec),
+            timestamp_to_isoformat(end_sec),
+        )
+
+        blocks_rewards = get_blocks_rewards(
+            withdrawals_target, start_datetime, end_datetime, limit=limit
+        )
+        return blocks_rewards
+
     def get_staking_proceeds(self, context: Context) -> int:
         """
         Retrieves a list of transactions, calculates MEV value and aggregates it with withdrawals.
@@ -31,23 +51,33 @@ def get_staking_proceeds(self, context: Context) -> int:
             context.epoch_details.end_block,
         )
 
-        if end_block is not None:
-            end_block -= 1
+        start_sec, end_sec = context.epoch_details.duration_range
+        no_blocks_to_get = context.epoch_details.no_blocks
 
+        blocks_rewards = self._retrieve_blocks_rewards(
+            start_sec, end_sec, withdrawals_target, limit=no_blocks_to_get
+        )
+
+        end_block_for_transactions = end_block - 1
         normal = get_transactions(
-            withdrawals_target, start_block, end_block, tx_type=AccountAction.NORMAL
+            withdrawals_target,
+            start_block,
+            end_block_for_transactions,
+            tx_type=AccountAction.NORMAL,
         )
         internal = get_transactions(
-            withdrawals_target, start_block, end_block, tx_type=AccountAction.INTERNAL
+            withdrawals_target,
+            start_block,
+            end_block_for_transactions,
+            tx_type=AccountAction.INTERNAL,
         )
         withdrawals = get_transactions(
             withdrawals_target,
             start_block,
-            end_block,
+            end_block_for_transactions,
             tx_type=AccountAction.BEACON_WITHDRAWAL,
         )
-
         mev_value = sum_mev(withdrawals_target, normal, internal)
         withdrawals_value = sum_withdrawals(withdrawals)
 
-        return aggregate_proceeds(mev_value, withdrawals_value)
+        return aggregate_proceeds(mev_value, withdrawals_value, blocks_rewards)

backend/app/settings.py

@@ -23,6 +23,8 @@ class Config(object):
     SUBGRAPH_ENDPOINT = os.getenv("SUBGRAPH_ENDPOINT")
     WEB3_PROVIDER = Web3.HTTPProvider(os.getenv("ETH_RPC_PROVIDER_URL"))
     ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")
+    BITQUERY_API_KEY = os.getenv("BITQUERY_API_KEY")
+    BITQUERY_BEARER = os.getenv("BITQUERY_BEARER")
     SCHEDULER_ENABLED = _parse_bool(os.getenv("SCHEDULER_ENABLED"))
     CACHE_TYPE = "SimpleCache"
 

backend/tests/conftest.py

@@ -9,6 +9,11 @@
 
 import gql
 import pytest
+from eth_account import Account
+from flask import g as request_context
+from flask.testing import FlaskClient
+from web3 import Web3
+
 from app import create_app
 from app.engine.user.effective_deposit import DepositEvent, EventType, UserDeposit
 from app.extensions import db, deposits, glm, gql_factory, w3
@@ -23,9 +28,6 @@
 from app.legacy.crypto.eip712 import build_allocations_eip712_data, sign
 from app.modules.dto import AccountFundsDTO
 from app.settings import DevConfig, TestConfig
-from eth_account import Account
-from flask import g as request_context
-from flask.testing import FlaskClient
 from tests.helpers.constants import (
     ALICE,
     ALL_INDIVIDUAL_REWARDS,
@@ -58,7 +60,6 @@
 from tests.helpers.mocked_epoch_details import EPOCH_EVENTS
 from tests.helpers.octant_rewards import octant_rewards
 from tests.helpers.subgraph.events import create_deposit_event
-from web3 import Web3
 
 # Contracts mocks
 MOCK_EPOCHS = MagicMock(spec=Epochs)
@@ -106,6 +107,34 @@ def mock_etherscan_api_get_block_num_from_ts(*args, **kwargs):
     return int(example_resp_json["result"])
 
 
+def mock_bitquery_api_get_blocks_rewards(*args, **kwargs):
+    example_resp_json = {
+        "data": {
+            "ethereum": {
+                "blocks": [
+                    {
+                        "timestamp": {"unixtime": 1708448963},
+                        "reward": 0.024473700594149782,
+                        "address": {
+                            "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326"
+                        },
+                    },
+                    {
+                        "timestamp": {"unixtime": 1708449035},
+                        "reward": 0.05342909432569912,
+                        "address": {
+                            "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326"
+                        },
+                    },
+                ]
+            }
+        }
+    }
+
+    blocks = example_resp_json["data"]["ethereum"]["blocks"]
+    return blocks
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--runapi",
@@ -512,6 +541,14 @@ def patch_etherscan_get_block_api(monkeypatch):
     )
 
 
+@pytest.fixture(scope="function")
+def patch_bitquery_get_blocks_rewards(monkeypatch):
+    monkeypatch.setattr(
+        "app.modules.staking.proceeds.service.aggregated.get_blocks_rewards",
+        mock_bitquery_api_get_blocks_rewards,
+    )
+
+
 @pytest.fixture(scope="function")
 def mock_users_db(app, user_accounts):
     alice = database.user.add_user(user_accounts[0].address)