forge/bench/

runner.rs

// #t(file: rust_alloc_in_loop) CLI binary, allocations are fine for readability.
// #t(file: rust_clone_in_loop) CLI binary, cloning strings in display loops is fine.

#![expect(
    clippy::cast_possible_truncation,
    clippy::cast_possible_wrap,
    clippy::cast_precision_loss,
    reason = "benchmark durations, iteration counts, and thread indices are bounded by CLI inputs and serialized report fields"
)]

//! Benchmark execution: discover specs, run simulations, collect results.

use anyhow::{Context, Result};
use rayon::prelude::*;
use wowlab_common::{output, sys, time::Instant};
use wowlab_engine_adapter_data::{LocalCsvResolver, OverlayResolver};
use wowlab_engine_application::{
    ApplicationError, ChunkRun, IntentOverrides, ResolvedSetup, build_handler_from_setup,
    resolve_setup, run_chunk, simulate_intent,
};
use wowlab_engine_ports::{ChunkAssignment, NoopProgress, SpecDescriptor, content_catalog};
use wowlab_fs::file;
use wowlab_types::{constants::MS_PER_SECOND, game::SpecId};

use super::types::{BenchRun, ScalingPoint, ScalingResult, SpecBenchResult};
use crate::{
    constants::{DEFAULT_SEED, default_data_dir, engine_dir},
    provider::ComparisonConfig,
    run::RunParameters,
    sim,
};

const DECIMAL_BASE: f64 = 10.0;
const THREAD_STEP_FACTOR: usize = 2;
const DECIMALS_IPS: u32 = 1;
const DECIMALS_MULTIPLIER: u32 = 2;
const DECIMALS_RATIO: u32 = 3;

fn round(v: f64, decimals: u32) -> f64 {
    let factor = DECIMAL_BASE.powi(decimals as i32);

    (v * factor).round() / factor
}

pub(super) fn discover_specs(filter: &[String]) -> Result<Vec<(&'static SpecDescriptor, String)>> {
    let rotation_dir = engine_dir().join("examples/rotations");
    let catalog = content_catalog().context("engine content catalog is unavailable")?;

    let mut all_dps: Vec<_> = catalog
        .descriptors()
        .iter()
        .filter(|d| d.spec_id.is_dps())
        .collect();

    all_dps.sort_by_key(|d| d.spec_id.slug());

    let mut specs = Vec::new();

    for desc in all_dps {
        let slug = desc.spec_id.slug();

        if !filter.is_empty() && !filter.iter().any(|f| f == slug) {
            continue;
        }

        // Skip specs without comparison-profile gear; gearless throughput is not comparable.

        if crate::simc::simc_profile_path(desc.spec_id).is_none() {
            continue;
        }

        let rotation_path = rotation_dir.join(format!("{slug}_assisted.json"));

        if let Ok(script) = file::read_text(&rotation_path) {
            specs.push((desc, script));
        }
    }

    Ok(specs)
}

pub(super) fn run_benchmarks(
    specs: &[(&'static SpecDescriptor, String)],
    parameters: RunParameters,
) -> Result<(Vec<SpecBenchResult>, tokio::runtime::Runtime)> {
    let iterations = parameters.iterations();
    let duration = parameters.fight_duration_secs();
    let data_dir = std::env::var("WOWLAB_DATA_DIR").unwrap_or_else(|_| default_data_dir());
    let rt = tokio::runtime::Runtime::new().context("failed to create tokio runtime")?;

    output::header(&format!(
        "Single-threaded benchmark: {} specs ({} iterations, {}s duration)",
        specs.len(),
        iterations,
        duration,
    ));
    output::blank();

    let mut results = Vec::new();
    let mut failed: Vec<(String, String)> = Vec::new();

    for (desc, rotation_script) in specs {
        let slug = desc.spec_id.slug();

        match bench_spec_single(&rt, desc, rotation_script, &data_dir, parameters) {
            Ok(result) => {
                output::success(&format!(
                    "{:<30} {:>10.0} iter/s  {:>10} DPS  ({}ms)",
                    result.slug,
                    result.iterations_per_sec,
                    output::fmt_number(result.mean_dps),
                    result.elapsed_ms,
                ));
                results.push(result);
            }
            Err(e) => {
                output::error(&format!("{slug:<30} FAILED: {e}"));
                failed.push((slug.to_string(), e.to_string()));
            }
        }
    }

    output::blank();

    if !failed.is_empty() {
        output::kv("Skipped", &format!("{} specs failed", failed.len()));
    }

    print_summary(&results);

    Ok((results, rt))
}

pub(super) fn run_scaling(
    rt: &tokio::runtime::Runtime,
    specs: &[(&'static SpecDescriptor, String)],
    single_results: &[SpecBenchResult],
    parameters: RunParameters,
) -> Vec<ScalingResult> {
    let catalog = match content_catalog() {
        Ok(catalog) => catalog,
        Err(error) => {
            output::error(&format!("engine content catalog is unavailable: {error}"));

            return Vec::new();
        }
    };
    let iterations = parameters.iterations();
    let data_dir = std::env::var("WOWLAB_DATA_DIR").unwrap_or_else(|_| default_data_dir());
    let profile = sys::os_profile();
    let thread_counts = compute_thread_counts(&profile);

    output::blank();
    output::header(&format!(
        "Thread scaling benchmark: {} specs, thread counts {:?}",
        specs.len(),
        thread_counts,
    ));
    output::blank();

    let mut scaling_results = Vec::new();

    for (desc, rotation_script) in specs {
        let slug = desc.spec_id.slug();

        if !single_results.iter().any(|r| r.slug == slug) {
            output::detail(&format!("{slug:<30} skipped (no single-threaded baseline)"));
            continue;
        }

        let rotation_id = format!("{slug}_assisted");

        let resolver = OverlayResolver::new(LocalCsvResolver::new(data_dir.as_str()))
            .with_rotation_script(&rotation_id, rotation_script.clone());
        let resolver_dyn = wowlab_engine_ports::DynDataResolver::from_ref(&resolver);
        let sim_config = match rt.block_on(bench_sim_config(
            desc.spec_id,
            &rotation_id,
            parameters,
            resolver_dyn,
        )) {
            Ok(c) => c,
            Err(e) => {
                output::error(&format!("{slug:<30} FAILED: {e}"));
                continue;
            }
        };
        let setup = match rt.block_on(resolve_setup(
            catalog,
            &sim_config,
            resolver_dyn,
            &IntentOverrides::default(),
        )) {
            Ok(s) => s,
            Err(e) => {
                output::error(&format!("{slug:<30} FAILED: {e}"));
                continue;
            }
        };

        let baseline_ips = single_results
            .iter()
            .find(|r| r.slug == slug)
            .map_or(1.0, |r| r.iterations_per_sec);

        output::subheader(&format!("{slug} (baseline: {baseline_ips:.0} iter/s)"));

        let mut points = Vec::with_capacity(thread_counts.len());
        let mut prev_ips = baseline_ips;

        for &n in &thread_counts {
            let start = Instant::now();
            // PANIC-BOUNDARY: isolate one scaling run so the CLI can report the failed spec.
            let par_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                run_scaling_dispatch(&setup, DEFAULT_SEED, parameters, n)
            }));

            match par_result {
                Ok(Err(e)) => {
                    output::error(&format!("  {n:>3} threads: FAILED: {e}"));
                    break;
                }
                Err(_panic) => {
                    output::error(&format!("  {n:>3} threads: PANICKED"));
                    break;
                }
                Ok(Ok(())) => {}
            }

            let elapsed_ms = start.elapsed().as_millis() as u64;

            let ips = if elapsed_ms > 0 {
                f64::from(iterations) / (elapsed_ms as f64 / MS_PER_SECOND)
            } else {
                0.0
            };
            let multiplier = ips / baseline_ips;
            let efficiency = multiplier / n as f64;
            let marginal_gain = if prev_ips > 0.0 {
                (ips - prev_ips) / prev_ips
            } else {
                0.0
            };

            output::detail(&format!(
                "  {n:>3} threads: {:>10.0} iter/s  {multiplier:>5.2}x  eff {:.0}%  marginal +{:.1}%",
                ips,
                efficiency * wowlab_types::constants::HUNDRED,
                marginal_gain * wowlab_types::constants::HUNDRED,
            ));

            points.push(ScalingPoint {
                threads: n,
                elapsed_ms,
                iterations_per_sec: round(ips, DECIMALS_IPS),
                multiplier: round(multiplier, DECIMALS_MULTIPLIER),
                efficiency: round(efficiency, DECIMALS_RATIO),
                marginal_gain: round(marginal_gain, DECIMALS_RATIO),
                is_p_core_range: n <= profile.p_cores,
            });

            prev_ips = ips;
        }

        scaling_results.push(ScalingResult {
            slug: slug.to_string(),
            iterations,
            points,
        });
    }

    scaling_results
}

fn compute_thread_counts(profile: &sys::OsProfile) -> Vec<usize> {
    let mut counts = vec![1_usize];
    let mut n = THREAD_STEP_FACTOR;

    while n < profile.p_cores {
        counts.push(n);
        n *= THREAD_STEP_FACTOR;
    }

    if profile.p_cores > 1 && !counts.contains(&profile.p_cores) {
        counts.push(profile.p_cores);
    }

    let p_plus_e = profile.p_cores + profile.e_cores;

    if profile.e_cores > 0 && p_plus_e != profile.p_cores {
        counts.push(p_plus_e);
    }

    if profile.logical_cores > p_plus_e && !counts.contains(&profile.logical_cores) {
        counts.push(profile.logical_cores);
    }

    counts.sort_unstable();
    counts.dedup();

    counts
}

fn run_scaling_dispatch(
    setup: &ResolvedSetup,
    seed_base: u64,
    parameters: RunParameters,
    num_threads: usize,
) -> Result<()> {
    let iterations = parameters.iterations();
    let pool = rayon::ThreadPoolBuilder::new()
        .num_threads(num_threads)
        .build()
        .context("Failed to create thread pool")?;

    let base_count = iterations / num_threads as u32;
    let remainder = iterations % num_threads as u32;

    let assignments: Vec<ChunkAssignment> = (0..num_threads)
        .map(|t| {
            let t_u32 = t as u32;
            let my_count = base_count + u32::from(t_u32 < remainder);
            let mut assignment = ChunkAssignment::single_indexed("scaling", t_u32, my_count);

            assignment.chunk_id = format!("scaling-{t}");

            assignment
        })
        .collect();

    let results: Vec<Result<_, ApplicationError>> = pool.install(|| {
        assignments
            .par_iter()
            .map(|assignment| {
                run_chunk(
                    ChunkRun {
                        assignment,
                        duration_s: setup.duration_s,
                        seed_base,
                    },
                    &NoopProgress,
                    || build_handler_from_setup(setup),
                )
            })
            .collect()
    });

    for r in results {
        r.context("Worker failed")?;
    }

    Ok(())
}

async fn bench_sim_config(
    spec: SpecId,
    rotation_id: &str,
    parameters: RunParameters,
    resolver: &wowlab_engine_ports::DynDataResolver<'_>,
) -> Result<String> {
    let config = ComparisonConfig {
        spec,
        parameters,
        race: None,
        bugs: true,
        simc_debug: false,
    };

    crate::wowlab::wowlab_profile_config(&config, rotation_id, resolver).await
}

fn bench_spec_single(
    rt: &tokio::runtime::Runtime,
    desc: &SpecDescriptor,
    rotation_script: &str,
    data_dir: &str,
    parameters: RunParameters,
) -> Result<SpecBenchResult> {
    let catalog = content_catalog().context("engine content catalog is unavailable")?;
    let iterations = parameters.iterations();
    let duration = parameters.fight_duration_secs();
    let slug = desc.spec_id.slug();
    let rotation_id = format!("{slug}_assisted");

    let resolver = OverlayResolver::new(LocalCsvResolver::new(data_dir))
        .with_rotation_script(&rotation_id, rotation_script.to_string());
    let resolver_dyn = wowlab_engine_ports::DynDataResolver::from_ref(&resolver);
    let sim_config = rt.block_on(bench_sim_config(
        desc.spec_id,
        &rotation_id,
        parameters,
        resolver_dyn,
    ))?;

    let chunk_label = format!("bench-{slug}");
    let chunk = sim::make_chunk("bench", iterations, Some(&chunk_label));

    let start = Instant::now();
    // PANIC-BOUNDARY: isolate one benchmark so the remaining spec results stay reportable.
    let sim_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
        rt.block_on(simulate_intent(
            catalog,
            &sim_config,
            &chunk,
            DEFAULT_SEED,
            resolver_dyn,
            &NoopProgress,
        ))
    }));
    let report = match sim_result {
        Ok(Ok(r)) => r,
        Ok(Err(e)) => anyhow::bail!("simulation error for {slug}: {e}"),
        Err(panic) => {
            let msg = panic
                .downcast_ref::<String>()
                .map(String::as_str)
                .or_else(|| panic.downcast_ref::<&str>().copied())
                .unwrap_or("unknown panic");

            anyhow::bail!("panic during {slug}: {msg}");
        }
    };
    let elapsed_ms = start.elapsed().as_millis() as u64;

    let telemetry = sim::decode_telemetry(&report)?;
    let mean_dps = sim::mean_dps(&telemetry);
    let iters_per_sec = if elapsed_ms > 0 {
        f64::from(iterations) / (elapsed_ms as f64 / MS_PER_SECOND)
    } else {
        0.0
    };

    Ok(SpecBenchResult {
        slug: slug.to_string(),
        iterations,
        duration_secs: duration,
        elapsed_ms,
        iterations_per_sec: round(iters_per_sec, DECIMALS_IPS),
        mean_dps: round(mean_dps, DECIMALS_IPS),
    })
}

fn print_summary(results: &[SpecBenchResult]) {
    let total_iters: u64 = results.iter().map(|r| u64::from(r.iterations)).sum();
    let total_ms: u64 = results.iter().map(|r| r.elapsed_ms).sum();
    let overall_ips = if total_ms > 0 {
        total_iters as f64 / (total_ms as f64 / MS_PER_SECOND)
    } else {
        0.0
    };

    output::kv("Total iterations", &output::fmt_number(total_iters as f64));
    output::kv("Total time", &format!("{total_ms}ms"));
    output::kv("Overall throughput", &format!("{overall_ips:.0} iter/s"));
}

pub(super) fn build_bench_run(
    results: Vec<SpecBenchResult>,
    scaling: Vec<ScalingResult>,
) -> BenchRun {
    BenchRun {
        engine_version: wowlab_engine::composition::EngineBuildMetadata::CURRENT
            .version()
            .to_string(),
        timestamp: shell_output("date", &["-u", "+%Y-%m-%dT%H:%M:%SZ"]),
        git_hash: shell_output("git", &["rev-parse", "--short", "HEAD"]),
        git_branch: shell_output("git", &["rev-parse", "--abbrev-ref", "HEAD"]),
        rustc_version: shell_output("rustc", &["--version"]),
        system: sys::os_profile(),
        results,
        scaling,
    }
}

fn shell_output(cmd: &str, args: &[&str]) -> String {
    let output = std::process::Command::new(cmd)
        .args(args)
        .output()
        .ok()
        .and_then(|o| String::from_utf8(o.stdout).ok());

    output.map_or_else(|| "unknown".to_string(), |s| s.trim().to_string())
}