diff --git a/Cargo.lock b/Cargo.lock index b68269b8b3..23c7f9d2e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -940,6 +940,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode 0.3.6", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "const-hex" version = "1.12.0" @@ -1333,6 +1346,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -2507,6 +2526,19 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -3020,6 +3052,12 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.36.4" @@ -3354,6 +3392,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "poseidon" version = "0.2.0" @@ -3436,7 +3480,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" dependencies = [ "csv", - "encode_unicode", + "encode_unicode 1.0.0", "is-terminal", "lazy_static", "term", @@ -4667,6 +4711,7 @@ dependencies = [ "anyhow", "bus-mapping", "clap", + "console", "ctor", "env_logger", "eth-types", @@ -4677,6 +4722,7 @@ dependencies = [ "halo2_proofs", "handlebars", "hex", + "indicatif", "itertools 0.11.0", "log", "mock", diff --git a/testool/Cargo.toml b/testool/Cargo.toml index d9967af5eb..43fbc9e17e 100644 --- a/testool/Cargo.toml +++ b/testool/Cargo.toml @@ -5,10 +5,22 @@ version.workspace = true edition.workspace = true license.workspace = true +[lib] +name = "testool" + +[[bin]] +name = "testool" +path = "src/bin/testool.rs" + +[[bin]] +name = "trace-gen" +path = "src/bin/trace-gen.rs" + [dependencies] anyhow.workspace = true bus-mapping = { path = "../bus-mapping" } clap = { version = "4.5", features = ["derive"] } +console = "0.15" env_logger.workspace = true eth-types = { path="../eth-types" } ethers-core.workspace = true @@ -17,6 +29,7 @@ external-tracer = { path="../external-tracer" } glob = "0.3" handlebars = "4.3" hex.workspace = true +indicatif = "0.17" sha3 = "0.10" log.workspace = true itertools.workspace = true diff --git a/testool/src/main.rs b/testool/src/bin/testool.rs similarity index 74% rename from testool/src/main.rs rename to testool/src/bin/testool.rs index eb15e5c0e9..fe315f096f 100644 --- a/testool/src/main.rs +++ b/testool/src/bin/testool.rs @@ -1,36 +1,22 @@ #![feature(lazy_cell)] -/// Execute the bytecode from an empty state and run the EVM and State circuits -mod abi; -mod compiler; -mod config; -mod statetest; -mod utils; - -use crate::{config::TestSuite, statetest::ResultLevel}; +//! Execute the bytecode from an empty state and run the EVM and State circuits + use anyhow::{bail, Result}; use clap::Parser; -use compiler::Compiler; -use config::Config; use log::info; -use statetest::{ - load_statetests_suite, run_statetests_suite, run_test, CircuitsConfig, Results, StateTest, -}; -use std::{ - collections::{HashMap, HashSet}, - env, - fs::File, - io::{BufRead, BufReader, Write}, - path::PathBuf, - time::SystemTime, -}; +use std::{collections::HashSet, path::PathBuf, time::SystemTime}; use strum_macros::EnumString; - -const REPORT_FOLDER: &str = "report"; -const CODEHASH_FILE: &str = "./codehash.txt"; -const TEST_IDS_FILE: &str = "./test_ids.txt"; - -#[macro_use] -extern crate prettytable; +use testool::{ + compiler::Compiler, + config::Config, + config::TestSuite, + load_tests, + statetest::{ + load_statetests_suite, run_statetests_suite, run_test, CircuitsConfig, ResultLevel, + Results, StateTest, + }, + utils, write_test_ids, CODEHASH_FILE, REPORT_FOLDER, +}; #[allow(non_camel_case_types)] #[derive(PartialEq, Parser, EnumString, Debug, Clone, Copy)] @@ -81,55 +67,17 @@ struct Args { /// Specify a file including test IDs to run these tests #[clap(long)] - test_ids: Option, + test_ids: Option, /// Specify a file excluding test IDs to run these tests #[clap(long)] - exclude_test_ids: Option, + exclude_test_ids: Option, /// Verbose #[clap(short, long)] v: bool, } -fn read_test_ids(file_path: &str) -> Result> { - let worker_index = env::var("WORKER_INDEX") - .ok() - .and_then(|val| val.parse::().ok()) - .expect("WORKER_INDEX not set"); - let total_workers = env::var("TOTAL_WORKERS") - .ok() - .and_then(|val| val.parse::().ok()) - .expect("TOTAL_WORKERS not set"); - info!("total workers: {total_workers}, worker index: {worker_index}"); - - info!("read_test_ids from {}", file_path); - let mut total_jobs = 0; - let test_ids = BufReader::new(File::open(file_path)?) - .lines() - .map(|r| r.map(|line| line.trim().to_string())) - .inspect(|_| total_jobs += 1) - .enumerate() - .filter_map(|(idx, line)| { - if idx % total_workers == worker_index { - Some(line) - } else { - None - } - }) - .collect::, std::io::Error>>()?; - - info!("read_test_ids {} of {total_jobs}", test_ids.len()); - Ok(test_ids) -} - -fn write_test_ids(test_ids: &[String]) -> Result<()> { - let mut fd = File::create(TEST_IDS_FILE)?; - fd.write_all(test_ids.join("\n").as_bytes())?; - - Ok(()) -} - fn run_single_test( test: StateTest, suite: TestSuite, @@ -212,31 +160,7 @@ fn go() -> Result<()> { // It is better to sue deterministic testing order. // If there is a list, follow list. // If not, order by test id. - if let Some(test_ids_path) = args.test_ids { - if args.exclude_test_ids.is_some() { - log::warn!("--exclude-test-ids is ignored"); - } - let test_ids = read_test_ids(&test_ids_path)?; - let id_to_test: HashMap<_, _> = state_tests - .iter() - .map(|t| (t.id.clone(), t.clone())) - .collect(); - state_tests.clear(); - state_tests.extend( - test_ids - .into_iter() - .filter_map(|test_id| id_to_test.get(&test_id).cloned()), - ); - } else { - // sorting with reversed id string to prevent similar tests go together, so that - // computing heavy tests will not trigger OOM. - if let Some(exclude_test_ids_path) = args.exclude_test_ids { - let buf = std::fs::read_to_string(exclude_test_ids_path)?; - let set = buf.lines().map(|s| s.trim()).collect::>(); - state_tests.retain(|t| !set.contains(t.id.as_str())); - } - state_tests.sort_by_key(|t| t.id.chars().rev().collect::()); - } + load_tests(&mut state_tests, args.test_ids, args.exclude_test_ids)?; if args.report { let git_hash = utils::current_git_commit()?; diff --git a/testool/src/bin/trace-gen.rs b/testool/src/bin/trace-gen.rs new file mode 100644 index 0000000000..414fff318d --- /dev/null +++ b/testool/src/bin/trace-gen.rs @@ -0,0 +1,156 @@ +use anyhow::bail; +use clap::Parser; +use console::{style, Emoji}; +use eth_types::l2_types::BlockTraceV2; +use eth_types::{ToBigEndian, U256}; +use indicatif::{HumanDuration, ProgressBar, ProgressStyle}; +use rayon::prelude::*; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use testool::statetest::StateTest; +use testool::{ + compiler::Compiler, config::Config, load_tests, statetest::executor::into_traceconfig, + statetest::load_statetests_suite, CODEHASH_FILE, +}; + +static LOOKING_GLASS: Emoji<'_, '_> = Emoji("🔍 ", ""); +static PAPER: Emoji<'_, '_> = Emoji("📃 ", ""); +static SPARKLE: Emoji<'_, '_> = Emoji("✨ ", ":-)"); + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + /// Suite (by default is "default") + #[clap(long, default_value = "default")] + suite: String, + + /// Specify a file including test IDs to run these tests + #[clap(long)] + test_ids: Option, + + /// Specify a file excluding test IDs to run these tests + #[clap(long)] + exclude_test_ids: Option, + + #[clap(long)] + out_dir: PathBuf, +} + +fn main() -> anyhow::Result<()> { + let started = Instant::now(); + + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("error")) + .format_timestamp(None) + .format_level(false) + .format_module_path(false) + .format_target(false) + .init(); + + let spinner_style = ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}")? + .tick_chars("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"); + + let args = Args::parse(); + + println!( + "{} {}Checking compiler...", + style("[1/4]").bold().dim(), + LOOKING_GLASS + ); + let compiler = Compiler::new(true, Some(PathBuf::from(CODEHASH_FILE)))?; + + println!( + "{} {}Loading config...", + style("[2/4]").bold().dim(), + LOOKING_GLASS + ); + let config = Config::load()?; + let suite = config.suite(&args.suite)?.clone(); + + let pb = ProgressBar::new_spinner(); + pb.enable_steady_tick(Duration::from_millis(80)); + pb.set_style(spinner_style.clone()); + pb.set_prefix(format!("[3/4] {}", PAPER)); + pb.set_message("Loading state suite..."); + let mut state_tests = load_statetests_suite(&suite, config, compiler)?; + load_tests(&mut state_tests, args.test_ids, args.exclude_test_ids)?; + pb.finish_and_clear(); + println!( + "{} {}Loading state suite, done. {} tests collected in {}", + style("[3/4]").bold().dim(), + PAPER, + state_tests.len(), + suite.paths.join(", ") + ); + + let out_dir = args.out_dir; + std::fs::create_dir_all(&out_dir)?; + let error_report = Arc::new(Mutex::new(File::create(out_dir.join("errors.log"))?)); + let pb = ProgressBar::new(state_tests.len() as u64); + pb.enable_steady_tick(Duration::from_millis(80)); + pb.set_style(spinner_style.clone()); + pb.set_prefix(format!("[4/4] {}", SPARKLE)); + pb.set_message("Generating traces..."); + state_tests.into_par_iter().for_each(|st| { + let error_report = error_report.clone(); + let id = st.id.clone(); + pb.set_message(st.id.clone()); + if let Err(e) = build_trace(st, &out_dir) { + let mut error_report = error_report.lock().unwrap(); + writeln!(error_report, "{}: {}", id, e).unwrap(); + pb.set_message(format!("ERROR in {}: {}", id, e)); + } + }); + pb.finish_and_clear(); + println!( + "{} {}Done in {}", + style("[4/4]").bold().dim(), + SPARKLE, + HumanDuration(started.elapsed()) + ); + Ok(()) +} + +fn build_trace(st: StateTest, out_dir: &Path) -> anyhow::Result<()> { + let (_, mut trace_config, _) = into_traceconfig(st.clone()); + + for (_, acc) in trace_config.accounts.iter_mut() { + if acc.balance.to_be_bytes()[0] != 0u8 { + acc.balance = U256::from(1u128 << 127); + } + } + + let block_trace = match (external_tracer::l2trace(&trace_config), st.exception) { + (Ok(res), false) => res, + (Ok(_), true) => bail!("expected exception"), + (Err(e), false) => Err(e)?, + (Err(_), true) => return Ok(()), + }; + let block_trace = BlockTraceV2::from(block_trace); + + let mut block_trace = serde_json::to_value(&block_trace)?; + + // remove coinbase extras + block_trace["coinbase"] + .as_object_mut() + .unwrap() + .retain(|k, _| k == "address"); + + // remove code hashes + let codes = block_trace["codes"].as_array_mut().unwrap(); + for code in codes.iter_mut() { + code.as_object_mut().unwrap().remove("hash"); + } + + // cleanup storage_trace + let storage_trace = block_trace["storageTrace"].as_object_mut().unwrap(); + storage_trace.remove("addressHashes"); + storage_trace.remove("storeKeyHashes"); + + let out_path = File::create(out_dir.join(format!("{}.json", st.id)))?; + serde_json::to_writer_pretty(out_path, &block_trace)?; + + Ok(()) +} diff --git a/testool/src/lib.rs b/testool/src/lib.rs new file mode 100644 index 0000000000..5d368bfb07 --- /dev/null +++ b/testool/src/lib.rs @@ -0,0 +1,95 @@ +#![feature(lazy_cell)] + +#[macro_use] +extern crate prettytable; + +use crate::statetest::StateTest; +use log::info; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fs::File; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; + +pub mod abi; +pub mod compiler; +pub mod config; +pub mod statetest; +pub mod utils; + +pub const REPORT_FOLDER: &str = "report"; +pub const CODEHASH_FILE: &str = "./codehash.txt"; +pub const TEST_IDS_FILE: &str = "./test_ids.txt"; + +pub fn read_test_ids>(file_path: P) -> anyhow::Result> { + let file_path = file_path.as_ref(); + let worker_index = env::var("WORKER_INDEX") + .ok() + .and_then(|val| val.parse::().ok()) + .expect("WORKER_INDEX not set"); + let total_workers = env::var("TOTAL_WORKERS") + .ok() + .and_then(|val| val.parse::().ok()) + .expect("TOTAL_WORKERS not set"); + info!("total workers: {total_workers}, worker index: {worker_index}"); + + info!("read_test_ids from {:?}", file_path); + let mut total_jobs = 0; + let test_ids = BufReader::new(File::open(file_path)?) + .lines() + .map(|r| r.map(|line| line.trim().to_string())) + .inspect(|_| total_jobs += 1) + .enumerate() + .filter_map(|(idx, line)| { + if idx % total_workers == worker_index { + Some(line) + } else { + None + } + }) + .collect::, std::io::Error>>()?; + + info!("read_test_ids {} of {total_jobs}", test_ids.len()); + Ok(test_ids) +} + +pub fn write_test_ids(test_ids: &[String]) -> anyhow::Result<()> { + let mut fd = File::create(TEST_IDS_FILE)?; + fd.write_all(test_ids.join("\n").as_bytes())?; + + Ok(()) +} + +pub fn load_tests( + state_tests: &mut Vec, + test_ids_path: Option, + exclude_test_ids_path: Option, +) -> anyhow::Result<()> { + if let Some(test_ids_path) = test_ids_path { + if exclude_test_ids_path.is_some() { + log::warn!("--exclude-test-ids is ignored"); + } + let test_ids = read_test_ids(test_ids_path)?; + let id_to_test: HashMap<_, _> = state_tests + .iter() + .map(|t| (t.id.clone(), t.clone())) + .collect(); + state_tests.clear(); + state_tests.extend( + test_ids + .into_iter() + .filter_map(|test_id| id_to_test.get(&test_id).cloned()), + ); + } else { + // sorting with reversed id string to prevent similar tests go together, so that + // computing heavy tests will not trigger OOM. + if let Some(exclude_test_ids_path) = exclude_test_ids_path { + let buf = std::fs::read_to_string(exclude_test_ids_path)?; + let set = buf.lines().map(|s| s.trim()).collect::>(); + state_tests.retain(|t| !set.contains(t.id.as_str())); + } + state_tests.sort_by_key(|t| t.id.chars().rev().collect::()); + } + + Ok(()) +} diff --git a/testool/src/statetest/executor.rs b/testool/src/statetest/executor.rs index 16a868ad70..2dcecc8d4c 100644 --- a/testool/src/statetest/executor.rs +++ b/testool/src/statetest/executor.rs @@ -159,7 +159,7 @@ fn check_post( Ok(()) } -fn into_traceconfig(st: StateTest) -> (String, TraceConfig, StateTestResult) { +pub fn into_traceconfig(st: StateTest) -> (String, TraceConfig, StateTestResult) { let tx_type = st.tx_type(); let tx = st.build_tx(); diff --git a/testool/src/statetest/mod.rs b/testool/src/statetest/mod.rs index dde5ee9ffd..9fe07eccae 100644 --- a/testool/src/statetest/mod.rs +++ b/testool/src/statetest/mod.rs @@ -1,4 +1,4 @@ -mod executor; +pub mod executor; mod json; mod parse; mod results; diff --git a/testool/src/statetest/parse.rs b/testool/src/statetest/parse.rs index c98ed65517..b72008c0a9 100644 --- a/testool/src/statetest/parse.rs +++ b/testool/src/statetest/parse.rs @@ -1,4 +1,4 @@ -use crate::{abi, Compiler}; +use crate::{abi, compiler::Compiler}; use anyhow::{bail, Context, Result}; use eth_types::{address, AccessList, AccessListItem, Address, Bytes, H256, U256}; use log::debug; diff --git a/testool/src/statetest/yaml.rs b/testool/src/statetest/yaml.rs index c97a865569..9b78617c24 100644 --- a/testool/src/statetest/yaml.rs +++ b/testool/src/statetest/yaml.rs @@ -2,7 +2,7 @@ use super::{ parse, spec::{AccountMatch, Env, StateTest, DEFAULT_BASE_FEE}, }; -use crate::{abi, utils::MainnetFork, Compiler}; +use crate::{abi, compiler::Compiler, utils::MainnetFork}; use anyhow::{anyhow, bail, Context, Result}; use eth_types::{geth_types::Account, Address, Bytes, H256, U256}; use ethers_core::{k256::ecdsa::SigningKey, utils::secret_key_to_address};