"""Phase-2 strategy search tests (docs/phase-2-llm-architecture.md 10 P3.2/P3.3). ``evaluate`` measures one strategy on one scenario; ``optimize`` searches the strategy space (coarse grid, retreat refinement, LLM candidates) and returns a ranked playbook; ``strategy_name`` renders deterministic labels; and ``format_playbook`` renders the playbook as aligned ASCII columns. These tests pin the contract: benchmark fields, label rendering, rank ordering, candidate deduplication, determinism, the ``max_total_runs`` cost bound, and the text renderer. Small n (runs <= 5) keeps the file fast and deterministic. """ from __future__ import annotations from pathlib import Path from typing import TYPE_CHECKING import pytest from pf1e_simulator.scenarios import ScenarioSpec, SideSpec from pf1e_simulator.search import evaluate, format_playbook, optimize, strategy_name from pf1e_simulator.strategy import StrategySpec if TYPE_CHECKING: from pf1e_simulator.search import PlaybookEntry ROOT = Path(__file__).resolve().parents[1] MAP = ROOT / "data" / "maps" / "sample_arena.yaml" GOBLIN = ROOT / "data" / "monsters" / "goblin.json" ORC = ROOT / "data" / "monsters" / "orc.json" RUNS = 5 SEED = 7 ROUND_CAP = 20 PARTY = "players" def _scenario() -> ScenarioSpec: return ScenarioSpec( name="goblins-vs-orcs", map_file=MAP, sides=( SideSpec( name="players", zone="A", combatant_files=(GOBLIN, GOBLIN, GOBLIN), ), SideSpec( name="monsters", zone="B", combatant_files=(ORC, ORC), ), ), round_cap=ROUND_CAP, ) @pytest.fixture(scope="module") def playbook() -> tuple[PlaybookEntry, ...]: return optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED) def test_evaluate_default_strategy() -> None: """evaluate returns a benchmark with sane danger metrics and win band.""" benchmark = evaluate( StrategySpec(), _scenario(), party=PARTY, runs=100, seed=SEED, ) assert benchmark.scenario_name == "goblins-vs-orcs" assert benchmark.strategy == StrategySpec() assert benchmark.danger.runs == 100 assert 0.0 <= benchmark.danger.p_survives <= 1.0 assert 0.0 <= benchmark.danger.p_tpk <= 1.0 assert benchmark.danger.expected_rounds > 0.0 assert set(benchmark.wins) == {"players", "monsters"} assert benchmark.draws >= 0 low, high = benchmark.win_rate_band assert 0.0 <= low <= high <= 1.0 def test_strategy_name() -> None: """Labels render non-default levers in field order; the name field is excluded.""" assert strategy_name(StrategySpec()) == "default" assert strategy_name(StrategySpec(target_priority="weakest_ac")) == "weakest_ac" assert ( strategy_name(StrategySpec(target_priority="weakest_ac", spell_policy="none")) == "weakest_ac, spell_policy=none" ) assert ( strategy_name( StrategySpec(target_priority="lowest_hp", retreat_hp_fraction=0.5) ) == "lowest_hp, retreat=0.5" ) assert ( strategy_name(StrategySpec(name="custom", use_charges=False)) == "use_charges=false" ) def test_optimize_small_ranks_and_default(playbook: tuple[PlaybookEntry, ...]) -> None: """A small optimize returns contiguous ranks ordered by the rank key.""" assert len(playbook) >= 24 assert [entry.rank for entry in playbook] == list(range(1, len(playbook) + 1)) assert any(entry.strategy == StrategySpec() for entry in playbook) keys = [ ( -entry.benchmark.danger.p_survives, entry.benchmark.danger.expected_hp_lost, -entry.benchmark.danger.worst_decile_party_hp, entry.benchmark.danger.expected_rounds, strategy_name(entry.strategy), ) for entry in playbook ] assert keys == sorted(keys) def test_optimize_deterministic() -> None: """Two optimize calls with the same inputs return identical playbooks.""" first = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED) second = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED) assert first == second def test_optimize_candidates_deduped() -> None: """LLM candidates run once each; duplicates and grid duplicates collapse.""" in_grid = StrategySpec(target_priority="weakest_ac", spell_policy="none") custom = StrategySpec( target_priority="lowest_hp", spell_policy="none", use_charges=False, use_class_abilities=False, retreat_hp_fraction=0.75, ) playbook = optimize( _scenario(), party=PARTY, runs=RUNS, seed=SEED, candidates=(custom, custom, in_grid), ) custom_matches = [entry for entry in playbook if entry.strategy == custom] assert len(custom_matches) == 1 in_grid_matches = [entry for entry in playbook if entry.strategy == in_grid] assert len(in_grid_matches) == 1 direct = evaluate(custom, _scenario(), party=PARTY, runs=RUNS, seed=SEED) assert custom_matches[0].benchmark.danger == direct.danger def test_budget_raises_below_required_total() -> None: """A max_total_runs below the required total raises before running.""" with pytest.raises(ValueError, match="budget") as excinfo: optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=1) assert "max_total_runs" in str(excinfo.value) def test_budget_generous_cap_raises_nothing() -> None: """A generous max_total_runs cap does not raise.""" playbook = optimize( _scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=10_000, ) assert len(playbook) >= 24 def test_format_playbook_columns_and_ascii( playbook: tuple[PlaybookEntry, ...], ) -> None: """The renderer shows header tokens, a dash separator, aligned rows, ASCII.""" text = format_playbook(playbook, PARTY) for token in ( "Rank", "Strategy", "Runs", "P(survive)", "P(TPK)", "P(>=1 death)", "HP lost", "Worst-decile HP", "Rounds", "Win% (3σ)", ): assert token in text lines = text.splitlines() assert len(lines) == 2 + len(playbook) separator = lines[1] assert separator assert set(separator) <= {"-", " "} assert len({len(line) for line in lines}) == 1 non_ascii = {char for char in text if ord(char) > 127} assert non_ascii <= {"σ"} top = playbook[0] rate = top.benchmark.wins.get(PARTY, 0) / top.benchmark.danger.runs low, high = top.benchmark.win_rate_band expected = f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]" assert expected in text