foundry_bench/
lib.rs

1use crate::results::{HyperfineOutput, HyperfineResult};
2use eyre::{Result, WrapErr};
3use foundry_common::{sh_eprintln, sh_println};
4use foundry_compilers::project_util::TempProject;
5use foundry_test_utils::util::clone_remote;
6use once_cell::sync::Lazy;
7use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
8use std::{
9    env,
10    path::{Path, PathBuf},
11    process::Command,
12    str::FromStr,
13};
14
15pub mod results;
16
17/// Default number of runs for benchmarks
18pub const RUNS: u32 = 5;
19
20/// Configuration for repositories to benchmark
21#[derive(Debug, Clone)]
22pub struct RepoConfig {
23    pub name: String,
24    pub org: String,
25    pub repo: String,
26    pub rev: String,
27}
28
29impl FromStr for RepoConfig {
30    type Err = eyre::Error;
31
32    fn from_str(spec: &str) -> Result<Self> {
33        // Split by ':' first to separate repo path from optional rev
34        let parts: Vec<&str> = spec.splitn(2, ':').collect();
35        let repo_path = parts[0];
36        let custom_rev = parts.get(1).copied();
37
38        // Now split the repo path by '/'
39        let path_parts: Vec<&str> = repo_path.split('/').collect();
40        if path_parts.len() != 2 {
41            eyre::bail!("Invalid repo format '{}'. Expected 'org/repo' or 'org/repo:rev'", spec);
42        }
43
44        let org = path_parts[0];
45        let repo = path_parts[1];
46
47        // Try to find this repo in BENCHMARK_REPOS to get the full config
48        let existing_config = BENCHMARK_REPOS.iter().find(|r| r.org == org && r.repo == repo);
49
50        let config = if let Some(existing) = existing_config {
51            // Use existing config but allow custom rev to override
52            let mut config = existing.clone();
53            if let Some(rev) = custom_rev {
54                config.rev = rev.to_string();
55            }
56            config
57        } else {
58            // Create new config with custom rev or default
59            // Name should follow the format: org-repo (with hyphen)
60            RepoConfig {
61                name: format!("{org}-{repo}"),
62                org: org.to_string(),
63                repo: repo.to_string(),
64                rev: custom_rev.unwrap_or("main").to_string(),
65            }
66        };
67
68        let _ = sh_println!("Parsed repo spec '{spec}' -> {config:?}");
69        Ok(config)
70    }
71}
72
73/// Available repositories for benchmarking
74pub fn default_benchmark_repos() -> Vec<RepoConfig> {
75    vec![
76        RepoConfig {
77            name: "ithacaxyz-account".to_string(),
78            org: "ithacaxyz".to_string(),
79            repo: "account".to_string(),
80            rev: "main".to_string(),
81        },
82        RepoConfig {
83            name: "solady".to_string(),
84            org: "Vectorized".to_string(),
85            repo: "solady".to_string(),
86            rev: "main".to_string(),
87        },
88    ]
89}
90
91// Keep a lazy static for compatibility
92pub static BENCHMARK_REPOS: Lazy<Vec<RepoConfig>> = Lazy::new(default_benchmark_repos);
93
94/// Foundry versions to benchmark
95///
96/// To add more versions for comparison, install them first:
97/// ```bash
98/// foundryup --install stable
99/// foundryup --install nightly
100/// foundryup --install v0.2.0  # Example specific version
101/// ```
102///
103/// Then add the version strings to this array. Supported formats:
104/// - "stable" - Latest stable release
105/// - "nightly" - Latest nightly build
106/// - "v0.2.0" - Specific version tag
107/// - "commit-hash" - Specific commit hash
108/// - "nightly-rev" - Nightly build with specific revision
109pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"];
110
111/// A benchmark project that represents a cloned repository ready for testing
112pub struct BenchmarkProject {
113    pub name: String,
114    pub temp_project: TempProject,
115    pub root_path: PathBuf,
116}
117
118impl BenchmarkProject {
119    /// Set up a benchmark project by cloning the repository
120    #[allow(unused_must_use)]
121    pub fn setup(config: &RepoConfig) -> Result<Self> {
122        let temp_project =
123            TempProject::dapptools().wrap_err("Failed to create temporary project")?;
124
125        // Get root path before clearing
126        let root_path = temp_project.root().to_path_buf();
127        let root = root_path.to_str().unwrap();
128
129        // Remove all files in the directory
130        for entry in std::fs::read_dir(&root_path)? {
131            let entry = entry?;
132            let path = entry.path();
133            if path.is_dir() {
134                std::fs::remove_dir_all(&path).ok();
135            } else {
136                std::fs::remove_file(&path).ok();
137            }
138        }
139
140        // Clone the repository
141        let repo_url = format!("https://github.com/{}/{}.git", config.org, config.repo);
142        clone_remote(&repo_url, root);
143
144        // Checkout specific revision if provided
145        if !config.rev.is_empty() && config.rev != "main" && config.rev != "master" {
146            let status = Command::new("git")
147                .current_dir(root)
148                .args(["checkout", &config.rev])
149                .status()
150                .wrap_err("Failed to checkout revision")?;
151
152            if !status.success() {
153                eyre::bail!("Git checkout failed for {}", config.name);
154            }
155        }
156
157        // Git submodules are already cloned via --recursive flag
158        // But npm dependencies still need to be installed
159        Self::install_npm_dependencies(&root_path)?;
160
161        sh_println!("  ✅ Project {} setup complete at {}", config.name, root);
162        Ok(BenchmarkProject { name: config.name.to_string(), root_path, temp_project })
163    }
164
165    /// Install npm dependencies if package.json exists
166    #[allow(unused_must_use)]
167    fn install_npm_dependencies(root: &Path) -> Result<()> {
168        if root.join("package.json").exists() {
169            sh_println!("  📦 Running npm install...");
170            let status = Command::new("npm")
171                .current_dir(root)
172                .args(["install"])
173                .stdout(std::process::Stdio::inherit())
174                .stderr(std::process::Stdio::inherit())
175                .status()
176                .wrap_err("Failed to run npm install")?;
177
178            if !status.success() {
179                sh_println!(
180                    "  ⚠️  Warning: npm install failed with exit code: {:?}",
181                    status.code()
182                );
183            } else {
184                sh_println!("  ✅ npm install completed successfully");
185            }
186        }
187        Ok(())
188    }
189
190    /// Run a command with hyperfine and return the results
191    ///
192    /// # Arguments
193    /// * `benchmark_name` - Name of the benchmark for organizing output
194    /// * `version` - Foundry version being benchmarked
195    /// * `command` - The command to benchmark
196    /// * `runs` - Number of runs to perform
197    /// * `setup` - Optional setup command to run before the benchmark series (e.g., "forge build")
198    /// * `conclude` - Optional conclude command to run after each timing run (e.g., cleanup)
199    /// * `verbose` - Whether to show command output
200    ///
201    /// # Hyperfine flags used:
202    /// * `--runs` - Number of timing runs
203    /// * `--setup` - Execute before the benchmark series (not before each run)
204    /// * `--conclude` - Execute after each timing run
205    /// * `--export-json` - Export results to JSON for parsing
206    /// * `--shell=bash` - Use bash for shell command execution
207    /// * `--show-output` - Show command output (when verbose)
208    #[allow(clippy::too_many_arguments)]
209    fn hyperfine(
210        &self,
211        benchmark_name: &str,
212        version: &str,
213        command: &str,
214        runs: u32,
215        setup: Option<&str>,
216        conclude: Option<&str>,
217        verbose: bool,
218    ) -> Result<HyperfineResult> {
219        // Create structured temp directory for JSON output
220        // Format: <temp_dir>/<benchmark_name>/<version>/<repo_name>/<benchmark_name>.json
221        let temp_dir = std::env::temp_dir();
222        let json_dir =
223            temp_dir.join("foundry-bench").join(benchmark_name).join(version).join(&self.name);
224        std::fs::create_dir_all(&json_dir)?;
225
226        let json_path = json_dir.join(format!("{benchmark_name}.json"));
227
228        // Build hyperfine command
229        let mut hyperfine_cmd = Command::new("hyperfine");
230        hyperfine_cmd
231            .current_dir(&self.root_path)
232            .arg("--runs")
233            .arg(runs.to_string())
234            .arg("--export-json")
235            .arg(&json_path)
236            .arg("--shell=bash");
237
238        // Add optional setup command
239        if let Some(setup_cmd) = setup {
240            hyperfine_cmd.arg("--setup").arg(setup_cmd);
241        }
242
243        // Add optional conclude command
244        if let Some(conclude_cmd) = conclude {
245            hyperfine_cmd.arg("--conclude").arg(conclude_cmd);
246        }
247
248        if verbose {
249            hyperfine_cmd.arg("--show-output");
250            hyperfine_cmd.stderr(std::process::Stdio::inherit());
251            hyperfine_cmd.stdout(std::process::Stdio::inherit());
252        }
253
254        // Add the benchmark command last
255        hyperfine_cmd.arg(command);
256
257        let status = hyperfine_cmd.status().wrap_err("Failed to run hyperfine")?;
258        if !status.success() {
259            eyre::bail!("Hyperfine failed for command: {}", command);
260        }
261
262        // Read and parse the JSON output
263        let json_content = std::fs::read_to_string(json_path)?;
264        let output: HyperfineOutput = serde_json::from_str(&json_content)?;
265
266        // Extract the first result (we only run one command at a time)
267        output.results.into_iter().next().ok_or_else(|| eyre::eyre!("No results from hyperfine"))
268    }
269
270    /// Benchmark forge test
271    pub fn bench_forge_test(
272        &self,
273        version: &str,
274        runs: u32,
275        verbose: bool,
276    ) -> Result<HyperfineResult> {
277        // Build before running tests
278        self.hyperfine(
279            "forge_test",
280            version,
281            "forge test",
282            runs,
283            Some("forge build"),
284            None,
285            verbose,
286        )
287    }
288
289    /// Benchmark forge build with cache
290    pub fn bench_forge_build_with_cache(
291        &self,
292        version: &str,
293        runs: u32,
294        verbose: bool,
295    ) -> Result<HyperfineResult> {
296        // No setup needed, uses existing cache
297        self.hyperfine("forge_build_with_cache", version, "forge build", runs, None, None, verbose)
298    }
299
300    /// Benchmark forge build without cache
301    pub fn bench_forge_build_no_cache(
302        &self,
303        version: &str,
304        runs: u32,
305        verbose: bool,
306    ) -> Result<HyperfineResult> {
307        // Clean before the benchmark series
308        self.hyperfine(
309            "forge_build_no_cache",
310            version,
311            "forge build",
312            runs,
313            Some("forge clean"),
314            None,
315            verbose,
316        )
317    }
318
319    /// Benchmark forge fuzz tests
320    pub fn bench_forge_fuzz_test(
321        &self,
322        version: &str,
323        runs: u32,
324        verbose: bool,
325    ) -> Result<HyperfineResult> {
326        // Build before running fuzz tests
327        self.hyperfine(
328            "forge_fuzz_test",
329            version,
330            r#"forge test --match-test "test[^(]*\([^)]+\)""#,
331            runs,
332            Some("forge build"),
333            None,
334            verbose,
335        )
336    }
337
338    /// Benchmark forge coverage
339    pub fn bench_forge_coverage(
340        &self,
341        version: &str,
342        runs: u32,
343        verbose: bool,
344    ) -> Result<HyperfineResult> {
345        // No setup needed, forge coverage builds internally
346        // Use --ir-minimum to avoid "Stack too deep" errors
347        self.hyperfine(
348            "forge_coverage",
349            version,
350            "forge coverage --ir-minimum",
351            runs,
352            None,
353            None,
354            verbose,
355        )
356    }
357
358    /// Get the root path of the project
359    pub fn root(&self) -> &Path {
360        &self.root_path
361    }
362
363    /// Run a specific benchmark by name
364    pub fn run(
365        &self,
366        benchmark: &str,
367        version: &str,
368        runs: u32,
369        verbose: bool,
370    ) -> Result<HyperfineResult> {
371        match benchmark {
372            "forge_test" => self.bench_forge_test(version, runs, verbose),
373            "forge_build_no_cache" => self.bench_forge_build_no_cache(version, runs, verbose),
374            "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose),
375            "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose),
376            "forge_coverage" => self.bench_forge_coverage(version, runs, verbose),
377            _ => eyre::bail!("Unknown benchmark: {}", benchmark),
378        }
379    }
380}
381
382/// Switch to a specific foundry version
383#[allow(unused_must_use)]
384pub fn switch_foundry_version(version: &str) -> Result<()> {
385    let output = Command::new("foundryup")
386        .args(["--use", version])
387        .output()
388        .wrap_err("Failed to run foundryup")?;
389
390    // Check if the error is about forge --version failing
391    let stderr = String::from_utf8_lossy(&output.stderr);
392    if stderr.contains("command failed") && stderr.contains("forge --version") {
393        eyre::bail!(
394            "Foundry binaries maybe corrupted. Please reinstall by running `foundryup --install <version>`"
395        );
396    }
397
398    if !output.status.success() {
399        sh_eprintln!("foundryup stderr: {stderr}");
400        eyre::bail!("Failed to switch to foundry version: {}", version);
401    }
402
403    sh_println!("  Successfully switched to version: {version}");
404    Ok(())
405}
406
407/// Get the current forge version
408pub fn get_forge_version() -> Result<String> {
409    let output = Command::new("forge")
410        .args(["--version"])
411        .output()
412        .wrap_err("Failed to get forge version")?;
413
414    if !output.status.success() {
415        eyre::bail!("forge --version failed");
416    }
417
418    let version =
419        String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?;
420
421    Ok(version.lines().next().unwrap_or("unknown").to_string())
422}
423
424/// Get the full forge version details including commit hash and date
425pub fn get_forge_version_details() -> Result<String> {
426    let output = Command::new("forge")
427        .args(["--version"])
428        .output()
429        .wrap_err("Failed to get forge version")?;
430
431    if !output.status.success() {
432        eyre::bail!("forge --version failed");
433    }
434
435    let full_output =
436        String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?;
437
438    // Extract relevant lines and format them
439    let lines: Vec<&str> = full_output.lines().collect();
440    if lines.len() >= 3 {
441        // Extract version, commit, and timestamp
442        let version = lines[0].trim();
443        let commit = lines[1].trim().replace("Commit SHA: ", "");
444        let timestamp = lines[2].trim().replace("Build Timestamp: ", "");
445
446        // Format as: "forge 1.2.3-nightly (51650ea 2025-06-27)"
447        let short_commit = &commit[..7]; // First 7 chars of commit hash
448        let date = timestamp.split('T').next().unwrap_or(&timestamp);
449
450        Ok(format!("{version} ({short_commit} {date})"))
451    } else {
452        // Fallback to just the first line if format is unexpected
453        Ok(lines.first().unwrap_or(&"unknown").to_string())
454    }
455}
456
457/// Get Foundry versions to benchmark from environment variable or default
458///
459/// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set,
460/// otherwise returns the default versions from FOUNDRY_VERSIONS constant.
461///
462/// The environment variable should be a comma-separated list of versions,
463/// e.g., "stable,nightly,v1.2.0"
464pub fn get_benchmark_versions() -> Vec<String> {
465    if let Ok(versions_env) = env::var("FOUNDRY_BENCH_VERSIONS") {
466        versions_env.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()
467    } else {
468        FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect()
469    }
470}
471
472/// Setup Repositories for benchmarking
473pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> {
474    // Check for FOUNDRY_BENCH_REPOS environment variable
475    let repos = if let Ok(repos_env) = env::var("FOUNDRY_BENCH_REPOS") {
476        // Parse repo specs from the environment variable
477        // Format should be: "org1/repo1,org2/repo2"
478        repos_env
479            .split(',')
480            .map(|s| s.trim())
481            .filter(|s| !s.is_empty())
482            .map(|s| s.parse::<RepoConfig>())
483            .collect::<Result<Vec<_>>>()
484            .expect("Failed to parse FOUNDRY_BENCH_REPOS")
485    } else {
486        BENCHMARK_REPOS.clone()
487    };
488
489    repos
490        .par_iter()
491        .map(|repo_config| {
492            let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project");
493            (repo_config.clone(), project)
494        })
495        .collect()
496}
foundry_bench/lib.rs

foundry_bench/
lib.rs
