feat(phase2): optimizer and scenario compiler (P3)

tests/test_phase2_search.py

@@ -0,0 +1,210 @@
+"""Phase-2 strategy search tests (docs/phase-2-llm-architecture.md 10 P3.2/P3.3).
+
+``evaluate`` measures one strategy on one scenario; ``optimize`` searches the
+strategy space (coarse grid, retreat refinement, LLM candidates) and returns a
+ranked playbook; ``strategy_name`` renders deterministic labels; and
+``format_playbook`` renders the playbook as aligned ASCII columns. These tests
+pin the contract: benchmark fields, label rendering, rank ordering, candidate
+deduplication, determinism, the ``max_total_runs`` cost bound, and the text
+renderer. Small n (runs <= 5) keeps the file fast and deterministic.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
+from pf1e_simulator.search import evaluate, format_playbook, optimize, strategy_name
+from pf1e_simulator.strategy import StrategySpec
+
+if TYPE_CHECKING:
+    from pf1e_simulator.search import PlaybookEntry
+
+ROOT = Path(__file__).resolve().parents[1]
+MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
+GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
+ORC = ROOT / "data" / "monsters" / "orc.json"
+
+RUNS = 5
+SEED = 7
+ROUND_CAP = 20
+PARTY = "players"
+
+
+def _scenario() -> ScenarioSpec:
+    return ScenarioSpec(
+        name="goblins-vs-orcs",
+        map_file=MAP,
+        sides=(
+            SideSpec(
+                name="players",
+                zone="A",
+                combatant_files=(GOBLIN, GOBLIN, GOBLIN),
+            ),
+            SideSpec(
+                name="monsters",
+                zone="B",
+                combatant_files=(ORC, ORC),
+            ),
+        ),
+        round_cap=ROUND_CAP,
+    )
+
+
+@pytest.fixture(scope="module")
+def playbook() -> tuple[PlaybookEntry, ...]:
+    return optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
+
+
+def test_evaluate_default_strategy() -> None:
+    """evaluate returns a benchmark with sane danger metrics and win band."""
+    benchmark = evaluate(
+        StrategySpec(),
+        _scenario(),
+        party=PARTY,
+        runs=100,
+        seed=SEED,
+    )
+
+    assert benchmark.scenario_name == "goblins-vs-orcs"
+    assert benchmark.strategy == StrategySpec()
+    assert benchmark.danger.runs == 100
+    assert 0.0 <= benchmark.danger.p_survives <= 1.0
+    assert 0.0 <= benchmark.danger.p_tpk <= 1.0
+    assert benchmark.danger.expected_rounds > 0.0
+    assert set(benchmark.wins) == {"players", "monsters"}
+    assert benchmark.draws >= 0
+    low, high = benchmark.win_rate_band
+    assert 0.0 <= low <= high <= 1.0
+
+
+def test_strategy_name() -> None:
+    """Labels render non-default levers in field order; the name field is excluded."""
+    assert strategy_name(StrategySpec()) == "default"
+    assert strategy_name(StrategySpec(target_priority="weakest_ac")) == "weakest_ac"
+    assert (
+        strategy_name(StrategySpec(target_priority="weakest_ac", spell_policy="none"))
+        == "weakest_ac, spell_policy=none"
+    )
+    assert (
+        strategy_name(
+            StrategySpec(target_priority="lowest_hp", retreat_hp_fraction=0.5)
+        )
+        == "lowest_hp, retreat=0.5"
+    )
+    assert (
+        strategy_name(StrategySpec(name="custom", use_charges=False))
+        == "use_charges=false"
+    )
+
+
+def test_optimize_small_ranks_and_default(playbook: tuple[PlaybookEntry, ...]) -> None:
+    """A small optimize returns contiguous ranks ordered by the rank key."""
+    assert len(playbook) >= 24
+    assert [entry.rank for entry in playbook] == list(range(1, len(playbook) + 1))
+    assert any(entry.strategy == StrategySpec() for entry in playbook)
+
+    keys = [
+        (
+            -entry.benchmark.danger.p_survives,
+            entry.benchmark.danger.expected_hp_lost,
+            -entry.benchmark.danger.worst_decile_party_hp,
+            entry.benchmark.danger.expected_rounds,
+            strategy_name(entry.strategy),
+        )
+        for entry in playbook
+    ]
+    assert keys == sorted(keys)
+
+
+def test_optimize_deterministic() -> None:
+    """Two optimize calls with the same inputs return identical playbooks."""
+    first = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
+    second = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
+    assert first == second
+
+
+def test_optimize_candidates_deduped() -> None:
+    """LLM candidates run once each; duplicates and grid duplicates collapse."""
+    in_grid = StrategySpec(target_priority="weakest_ac", spell_policy="none")
+    custom = StrategySpec(
+        target_priority="lowest_hp",
+        spell_policy="none",
+        use_charges=False,
+        use_class_abilities=False,
+        retreat_hp_fraction=0.75,
+    )
+    playbook = optimize(
+        _scenario(),
+        party=PARTY,
+        runs=RUNS,
+        seed=SEED,
+        candidates=(custom, custom, in_grid),
+    )
+
+    custom_matches = [entry for entry in playbook if entry.strategy == custom]
+    assert len(custom_matches) == 1
+    in_grid_matches = [entry for entry in playbook if entry.strategy == in_grid]
+    assert len(in_grid_matches) == 1
+
+    direct = evaluate(custom, _scenario(), party=PARTY, runs=RUNS, seed=SEED)
+    assert custom_matches[0].benchmark.danger == direct.danger
+
+
+def test_budget_raises_below_required_total() -> None:
+    """A max_total_runs below the required total raises before running."""
+    with pytest.raises(ValueError, match="budget") as excinfo:
+        optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=1)
+    assert "max_total_runs" in str(excinfo.value)
+
+
+def test_budget_generous_cap_raises_nothing() -> None:
+    """A generous max_total_runs cap does not raise."""
+    playbook = optimize(
+        _scenario(),
+        party=PARTY,
+        runs=RUNS,
+        seed=SEED,
+        max_total_runs=10_000,
+    )
+    assert len(playbook) >= 24
+
+
+def test_format_playbook_columns_and_ascii(
+    playbook: tuple[PlaybookEntry, ...],
+) -> None:
+    """The renderer shows header tokens, a dash separator, aligned rows, ASCII."""
+    text = format_playbook(playbook, PARTY)
+
+    for token in (
+        "Rank",
+        "Strategy",
+        "Runs",
+        "P(survive)",
+        "P(TPK)",
+        "P(>=1 death)",
+        "HP lost",
+        "Worst-decile HP",
+        "Rounds",
+        "Win% (3σ)",
+    ):
+        assert token in text
+
+    lines = text.splitlines()
+    assert len(lines) == 2 + len(playbook)
+    separator = lines[1]
+    assert separator
+    assert set(separator) <= {"-", " "}
+    assert len({len(line) for line in lines}) == 1
+
+    non_ascii = {char for char in text if ord(char) > 127}
+    assert non_ascii <= {"σ"}
+
+    top = playbook[0]
+    rate = top.benchmark.wins.get(PARTY, 0) / top.benchmark.danger.runs
+    low, high = top.benchmark.win_rate_band
+    expected = f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]"
+    assert expected in text