Sitelet https://github.com/init4tech/builder/commit/48890cfe79971fa3359f9bf9dae7549908c17ad9
Skip to content

Commit 48890cf

Browse files
Fraser999Fraser999
authored
perf: add benchmarks for sim_round (#242)
1 parent 91132a5 commit 48890cf

10 files changed

Lines changed: 1128 additions & 239 deletions

File tree

‎Cargo.lock‎

Lines changed: 317 additions & 189 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,15 @@ url = "2.5.4"
6666
[dev-dependencies]
6767
alloy-hardforks = "0.4.0"
6868
alloy-chains = "0.2"
69+
criterion = { version = "0.8.2", features = ["async_tokio"] }
6970
signet-bundle = "0.16.0-rc.11"
7071

72+
[[bench]]
73+
name = "sim"
74+
path = "benches/sim/main.rs"
75+
harness = false
76+
required-features = ["test-utils"]
77+
7178
# comment / uncomment for local dev
7279
# [patch.crates-io]
7380
# signet-constants = { path = "../signet-sdk/crates/constants" }

‎benches/sim/bundles.rs‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
//! Benchmarks for bundle simulation: varying counts, DB latency, and concurrency.
2+
3+
use crate::fixture::{Fixture, LATENCIES, set_up_bundle_sim, set_up_sender_distribution_sim};
4+
use builder::test_utils::setup_test_config;
5+
use criterion::{BatchSize, Bencher, BenchmarkId, Criterion};
6+
use std::time::Duration;
7+
8+
pub fn bench_bundle_counts(criterion: &mut Criterion) {
9+
const LATENCY: Duration = Duration::ZERO;
10+
const CONCURRENCY: usize = 4;
11+
12+
setup_test_config();
13+
let runtime = tokio::runtime::Runtime::new().unwrap();
14+
15+
let mut group = criterion.benchmark_group("sim_varying_simple_bundle_counts_no_db_latency");
16+
group.sample_size(10);
17+
18+
for bundle_count in [1, 10, 100, 1000] {
19+
group.bench_with_input(
20+
BenchmarkId::new("bundle_count", bundle_count),
21+
&bundle_count,
22+
|bench: &mut Bencher, &bundle_count| {
23+
bench.to_async(&runtime).iter_batched(
24+
|| set_up_bundle_sim(bundle_count, LATENCY, CONCURRENCY),
25+
Fixture::run,
26+
BatchSize::PerIteration,
27+
);
28+
},
29+
);
30+
}
31+
group.finish();
32+
}
33+
34+
pub fn bench_bundle_db_latency(criterion: &mut Criterion) {
35+
const BUNDLE_COUNT: usize = 10;
36+
const CONCURRENCY: usize = 4;
37+
38+
setup_test_config();
39+
let runtime = tokio::runtime::Runtime::new().unwrap();
40+
41+
let mut group =
42+
criterion.benchmark_group(format!("sim_{BUNDLE_COUNT}_simple_bundles_varying_db_latency"));
43+
group.sample_size(10);
44+
45+
for (label, latency) in LATENCIES {
46+
group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| {
47+
bench.to_async(&runtime).iter_batched(
48+
|| set_up_bundle_sim(BUNDLE_COUNT, lat, CONCURRENCY),
49+
Fixture::run,
50+
BatchSize::PerIteration,
51+
);
52+
});
53+
}
54+
group.finish();
55+
}
56+
57+
pub fn bench_bundle_concurrency(criterion: &mut Criterion) {
58+
const BUNDLE_COUNT: usize = 100;
59+
const LATENCY: Duration = Duration::ZERO;
60+
61+
setup_test_config();
62+
let runtime = tokio::runtime::Runtime::new().unwrap();
63+
64+
let mut group = criterion.benchmark_group(format!(
65+
"sim_{BUNDLE_COUNT}_simple_bundles_no_db_latency_varying_concurrency"
66+
));
67+
group.sample_size(10);
68+
69+
for concurrency in [1, 2, 4, 8, 16] {
70+
group.bench_with_input(
71+
BenchmarkId::new("thread_count", concurrency),
72+
&concurrency,
73+
|bench, &conc| {
74+
bench.to_async(&runtime).iter_batched(
75+
|| set_up_bundle_sim(BUNDLE_COUNT, LATENCY, conc),
76+
Fixture::run,
77+
BatchSize::PerIteration,
78+
);
79+
},
80+
);
81+
}
82+
group.finish();
83+
}
84+
85+
pub fn bench_bundle_sender_distribution(criterion: &mut Criterion) {
86+
const BUNDLE_COUNT: usize = 100;
87+
const CONCURRENCY: usize = 4;
88+
89+
setup_test_config();
90+
let runtime = tokio::runtime::Runtime::new().unwrap();
91+
92+
let mut group = criterion.benchmark_group(format!(
93+
"sim_{BUNDLE_COUNT}_simple_bundles_no_db_latency_varying_sender_count"
94+
));
95+
group.sample_size(10);
96+
97+
for num_senders in [1, 5, 20, 100] {
98+
group.bench_with_input(
99+
BenchmarkId::new("sender_count", num_senders),
100+
&num_senders,
101+
|bench, &senders| {
102+
bench.to_async(&runtime).iter_batched(
103+
|| set_up_sender_distribution_sim(BUNDLE_COUNT, senders, CONCURRENCY),
104+
Fixture::run,
105+
BatchSize::PerIteration,
106+
);
107+
},
108+
);
109+
}
110+
group.finish();
111+
}

‎benches/sim/complex_bundles.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
//! Benchmarks for complex bundle shapes: multi-tx bundles and bundles with host transactions.
2+
3+
use crate::fixture::{Fixture, set_up_host_tx_bundle_sim, set_up_multi_tx_bundle_sim};
4+
use builder::test_utils::setup_test_config;
5+
use criterion::{BatchSize, BenchmarkId, Criterion};
6+
7+
pub fn bench_multi_tx_bundles(criterion: &mut Criterion) {
8+
const BUNDLE_COUNT: usize = 100;
9+
const CONCURRENCY: usize = 4;
10+
11+
setup_test_config();
12+
let runtime = tokio::runtime::Runtime::new().unwrap();
13+
14+
let mut group = criterion.benchmark_group(format!(
15+
"sim_{BUNDLE_COUNT}_bundles_no_db_latency_varying_txs_per_bundle"
16+
));
17+
group.sample_size(10);
18+
19+
for txs_per_bundle in [1, 3, 5, 10] {
20+
group.bench_with_input(
21+
BenchmarkId::new("txs_per_bundle", txs_per_bundle),
22+
&txs_per_bundle,
23+
|bench, &tpb| {
24+
bench.to_async(&runtime).iter_batched(
25+
|| set_up_multi_tx_bundle_sim(BUNDLE_COUNT, tpb, CONCURRENCY),
26+
Fixture::run,
27+
BatchSize::PerIteration,
28+
);
29+
},
30+
);
31+
}
32+
group.finish();
33+
}
34+
35+
pub fn bench_host_tx_bundles(criterion: &mut Criterion) {
36+
const BUNDLE_COUNT: usize = 100;
37+
const CONCURRENCY: usize = 4;
38+
39+
setup_test_config();
40+
let runtime = tokio::runtime::Runtime::new().unwrap();
41+
42+
let mut group = criterion.benchmark_group(format!(
43+
"sim_{BUNDLE_COUNT}_bundles_no_db_latency_varying_host_txs_per_bundle"
44+
));
45+
group.sample_size(10);
46+
47+
for host_txs in [0, 1, 3, 5] {
48+
group.bench_with_input(
49+
BenchmarkId::new("host_txs_per_bundle", host_txs),
50+
&host_txs,
51+
|bench, &htpb| {
52+
bench.to_async(&runtime).iter_batched(
53+
|| set_up_host_tx_bundle_sim(BUNDLE_COUNT, htpb, CONCURRENCY),
54+
Fixture::run,
55+
BatchSize::PerIteration,
56+
);
57+
},
58+
);
59+
}
60+
group.finish();
61+
}

‎benches/sim/failed_bundles.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
//! Benchmarks for bundles that fail: preflight validation failures and EVM execution reverts.
2+
3+
use crate::fixture::{
4+
Fixture, LATENCIES, set_up_execution_failure_sim, set_up_preflight_failure_sim,
5+
};
6+
use builder::test_utils::setup_test_config;
7+
use criterion::{BatchSize, BenchmarkId, Criterion};
8+
9+
pub fn bench_preflight_failure(criterion: &mut Criterion) {
10+
const COUNT: usize = 10;
11+
const CONCURRENCY: usize = 4;
12+
13+
setup_test_config();
14+
let runtime = tokio::runtime::Runtime::new().unwrap();
15+
16+
let mut group =
17+
criterion.benchmark_group(format!("sim_{COUNT}_simple_bundles_all_failing_preflight"));
18+
group.sample_size(10);
19+
20+
for (label, latency) in LATENCIES {
21+
group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| {
22+
bench.to_async(&runtime).iter_batched(
23+
|| set_up_preflight_failure_sim(COUNT, lat, CONCURRENCY),
24+
Fixture::run,
25+
BatchSize::PerIteration,
26+
);
27+
});
28+
}
29+
group.finish();
30+
}
31+
32+
pub fn bench_execution_failure(criterion: &mut Criterion) {
33+
const COUNT: usize = 10;
34+
const CONCURRENCY: usize = 4;
35+
36+
setup_test_config();
37+
let runtime = tokio::runtime::Runtime::new().unwrap();
38+
39+
let mut group =
40+
criterion.benchmark_group(format!("sim_{COUNT}_simple_bundles_all_failing_execution"));
41+
group.sample_size(10);
42+
43+
for (label, latency) in LATENCIES {
44+
group.bench_with_input(BenchmarkId::new("latency", label), &latency, |bench, &lat| {
45+
bench.to_async(&runtime).iter_batched(
46+
|| set_up_execution_failure_sim(COUNT, lat, CONCURRENCY),
47+
Fixture::run,
48+
BatchSize::PerIteration,
49+
);
50+
});
51+
}
52+
group.finish();
53+
}

0 commit comments

Comments
 (0)