Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/improve fee history performance #5182

Merged
merged 43 commits into from
Nov 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
d025d82
add structures which implements cache for base_fee_per_gas and used_r…
allnil Oct 23, 2023
a4b9195
prepare blueprint for cache integration for fees values
allnil Oct 24, 2023
558b3c4
add BlockFees type to collect fees data for particular block in cache…
allnil Oct 25, 2023
54204b4
format and clean
allnil Oct 25, 2023
2f40820
add comment
allnil Oct 25, 2023
acbbd76
leave comment regarding collected values
allnil Oct 25, 2023
43a4d4a
delete unused structs
allnil Oct 25, 2023
70dfb48
remove unused imports
allnil Oct 25, 2023
e690f42
remove fee history cache from eth_state, prepare separate type and ca…
allnil Oct 26, 2023
a9b0740
fix tests for single block
allnil Oct 27, 2023
dae2964
add parent_hash in mock data for tests
allnil Oct 27, 2023
52d4156
BTreeMap wrapped with RwLock, prepare task which listens to new canon…
allnil Oct 30, 2023
93ed179
add atomic bounds, refactor cache and provider
allnil Oct 31, 2023
1ae2aae
handle empty block intervals and fee history
allnil Nov 1, 2023
b3c2d8d
prepare ground for futher refactoring, remove provider from FeeHistor…
allnil Nov 3, 2023
e4ade22
refactor logic to use iterator over blocks in on_new_blocks function,…
allnil Nov 4, 2023
1d2f455
Merge branch 'main' of github.com:allnil/reth into feat/improve_fee_h…
allnil Nov 4, 2023
0cde2a1
merge
allnil Nov 4, 2023
912fd0f
prepare type to store fee data
allnil Nov 4, 2023
2805a7a
add annotation for clippy to ignore too many arguments on new fn
allnil Nov 4, 2023
30d0b4c
merge with main
allnil Nov 6, 2023
bbe4483
Merge branch 'main' of github.com:allnil/reth into feat/improve_fee_h…
allnil Nov 14, 2023
398e885
refactor, precalculate percentiles, rewards
allnil Nov 15, 2023
7d072b5
merge
allnil Nov 15, 2023
e213d13
add mock for block_range, update tests
allnil Nov 16, 2023
811a3c8
Merge branch 'main' of github.com:allnil/reth into feat/improve_fee_h…
allnil Nov 16, 2023
08a25fc
add simple test on rewards, calculate rewards on the fly
allnil Nov 16, 2023
ff3aed8
remove test because of transactions, for now
allnil Nov 16, 2023
1bc833a
merge with main
allnil Nov 16, 2023
2f61a8c
Update crates/rpc/rpc/src/eth/api/mod.rs
allnil Nov 20, 2023
66bb10d
merge and fmt
allnil Nov 20, 2023
30cd2ce
Merge branch 'feat/improve_fee_history_performance' of github.com:all…
allnil Nov 20, 2023
b131559
fix docs, add resolution in config, improve naming, move from blocks …
allnil Nov 20, 2023
b904b37
apply nightly rules
allnil Nov 20, 2023
3d4efe7
set proper and idiomatic semicolons
allnil Nov 20, 2023
8e9328c
move types adjacent to fee history to separate file
allnil Nov 22, 2023
9660e55
Merge branch 'main' into feat/improve_fee_history_performance
mattsse Nov 24, 2023
83f9c88
chore: touchups
mattsse Nov 24, 2023
3cbd463
refactor
mattsse Nov 24, 2023
7dc0e83
wip: caching
mattsse Nov 24, 2023
710540e
Merge branch 'main' into feat/improve_fee_history_performance
mattsse Nov 27, 2023
87bd429
cleanup
mattsse Nov 27, 2023
ef27a2f
cleanuo
mattsse Nov 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/net/downloaders/src/test_utils/bodies_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ impl BodiesClient for TestBodiesClient {

Box::pin(async move {
if should_respond_empty {
return Ok((PeerId::default(), vec![]).into());
return Ok((PeerId::default(), vec![]).into())
}

if should_delay {
Expand Down
2 changes: 1 addition & 1 deletion crates/net/eth-wire/src/capability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ impl SharedCapability {
/// Returns an error if the offset is equal or less than [`MAX_RESERVED_MESSAGE_ID`].
pub(crate) fn new(name: &str, version: u8, offset: u8) -> Result<Self, SharedCapabilityError> {
if offset <= MAX_RESERVED_MESSAGE_ID {
return Err(SharedCapabilityError::ReservedMessageIdOffset(offset));
return Err(SharedCapabilityError::ReservedMessageIdOffset(offset))
}

match name {
Expand Down
4 changes: 2 additions & 2 deletions crates/primitives/src/snapshot/segment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,15 +117,15 @@ impl SnapshotSegment {
) -> Option<(Self, RangeInclusive<BlockNumber>, RangeInclusive<TxNumber>)> {
let mut parts = name.to_str()?.split('_');
if parts.next() != Some("snapshot") {
return None;
return None
}

let segment = Self::from_str(parts.next()?).ok()?;
let (block_start, block_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?);
let (tx_start, tx_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?);

if block_start >= block_end || tx_start > tx_end {
return None;
return None
}

Some((segment, block_start..=block_end, tx_start..=tx_end))
Expand Down
10 changes: 9 additions & 1 deletion crates/rpc/rpc-builder/src/auth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ use reth_provider::{
StateProviderFactory,
};
use reth_rpc::{
eth::{cache::EthStateCache, gas_oracle::GasPriceOracle, EthFilterConfig},
eth::{
cache::EthStateCache, gas_oracle::GasPriceOracle, EthFilterConfig, FeeHistoryCache,
FeeHistoryCacheConfig,
},
AuthLayer, BlockingTaskPool, Claims, EngineEthApi, EthApi, EthFilter,
EthSubscriptionIdProvider, JwtAuthValidator, JwtSecret,
};
Expand Down Expand Up @@ -57,7 +60,11 @@ where
// spawn a new cache task
let eth_cache =
EthStateCache::spawn_with(provider.clone(), Default::default(), executor.clone());

let gas_oracle = GasPriceOracle::new(provider.clone(), Default::default(), eth_cache.clone());

let fee_history_cache =
FeeHistoryCache::new(eth_cache.clone(), FeeHistoryCacheConfig::default());
let eth_api = EthApi::with_spawner(
provider.clone(),
pool.clone(),
Expand All @@ -67,6 +74,7 @@ where
EthConfig::default().rpc_gas_cap,
Box::new(executor.clone()),
BlockingTaskPool::build().expect("failed to build tracing pool"),
fee_history_cache,
);
let config = EthFilterConfig::default()
.max_logs_per_response(DEFAULT_MAX_LOGS_PER_RESPONSE)
Expand Down
5 changes: 4 additions & 1 deletion crates/rpc/rpc-builder/src/eth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use reth_rpc::{
eth::{
cache::{EthStateCache, EthStateCacheConfig},
gas_oracle::GasPriceOracleConfig,
EthFilterConfig, RPC_DEFAULT_GAS_CAP,
EthFilterConfig, FeeHistoryCacheConfig, RPC_DEFAULT_GAS_CAP,
},
BlockingTaskPool, EthApi, EthFilter, EthPubSub,
};
Expand Down Expand Up @@ -46,6 +46,8 @@ pub struct EthConfig {
///
/// Sets TTL for stale filters
pub stale_filter_ttl: std::time::Duration,
/// Settings for the fee history cache
pub fee_history_cache: FeeHistoryCacheConfig,
}

impl EthConfig {
Expand All @@ -71,6 +73,7 @@ impl Default for EthConfig {
max_logs_per_response: DEFAULT_MAX_LOGS_PER_RESPONSE,
rpc_gas_cap: RPC_DEFAULT_GAS_CAP.into(),
stale_filter_ttl: DEFAULT_STALE_FILTER_TTL,
fee_history_cache: FeeHistoryCacheConfig::default(),
}
}
}
Expand Down
22 changes: 21 additions & 1 deletion crates/rpc/rpc-builder/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,9 @@ use reth_provider::{
use reth_rpc::{
eth::{
cache::{cache_new_blocks_task, EthStateCache},
fee_history_cache_new_blocks_task,
gas_oracle::GasPriceOracle,
EthBundle,
EthBundle, FeeHistoryCache,
},
AdminApi, AuthLayer, BlockingTaskGuard, BlockingTaskPool, Claims, DebugApi, EngineEthApi,
EthApi, EthFilter, EthPubSub, EthSubscriptionIdProvider, JwtAuthValidator, JwtSecret, NetApi,
Expand Down Expand Up @@ -1153,6 +1154,10 @@ where
}

/// Creates the [EthHandlers] type the first time this is called.
///
/// This will spawn the required service tasks for [EthApi] for:
/// - [EthStateCache]
/// - [FeeHistoryCache]
fn with_eth<F, R>(&mut self, f: F) -> R
where
F: FnOnce(&EthHandlers<Provider, Pool, Network, Events>) -> R,
Expand All @@ -1170,13 +1175,27 @@ where
);
let new_canonical_blocks = self.events.canonical_state_stream();
let c = cache.clone();

self.executor.spawn_critical(
"cache canonical blocks task",
Box::pin(async move {
cache_new_blocks_task(c, new_canonical_blocks).await;
}),
);

let fee_history_cache =
FeeHistoryCache::new(cache.clone(), self.config.eth.fee_history_cache.clone());
let new_canonical_blocks = self.events.canonical_state_stream();
let fhc = fee_history_cache.clone();
let provider_clone = self.provider.clone();
self.executor.spawn_critical(
"cache canonical blocks for fee history task",
Box::pin(async move {
fee_history_cache_new_blocks_task(fhc, new_canonical_blocks, provider_clone)
.await;
}),
);

let executor = Box::new(self.executor.clone());
let blocking_task_pool =
BlockingTaskPool::build().expect("failed to build tracing pool");
Expand All @@ -1189,6 +1208,7 @@ where
self.config.eth.rpc_gas_cap,
executor.clone(),
blocking_task_pool.clone(),
fee_history_cache,
);
let filter = EthFilter::new(
self.provider.clone(),
Expand Down
4 changes: 2 additions & 2 deletions crates/rpc/rpc-types/src/serde_helpers/json_u256.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,11 @@ where
} else {
// We could try to convert to a u128 here but there would probably be loss of
// precision, so we just return an error.
return Err(Error::custom("Deserializing a large non-mainnet TTD is not supported"));
return Err(Error::custom("Deserializing a large non-mainnet TTD is not supported"))
}
} else {
// must be i64 - negative numbers are not supported
return Err(Error::custom("Negative TTD values are invalid and will not be deserialized"));
return Err(Error::custom("Negative TTD values are invalid and will not be deserialized"))
};

Ok(num)
Expand Down
Loading
Loading