pf1e-simulator/tests/test_phase2_search.py

"""Phase-2 strategy search tests (docs/phase-2-llm-architecture.md 10 P3.2/P3.3).

``evaluate`` measures one strategy on one scenario; ``optimize`` searches the
strategy space (coarse grid, retreat refinement, LLM candidates) and returns a
ranked playbook; ``strategy_name`` renders deterministic labels; and
``format_playbook`` renders the playbook as aligned ASCII columns. These tests
pin the contract: benchmark fields, label rendering, rank ordering, candidate
deduplication, determinism, the ``max_total_runs`` cost bound, and the text
renderer. Small n (runs <= 5) keeps the file fast and deterministic.
"""

from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING

import pytest

from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
from pf1e_simulator.search import evaluate, format_playbook, optimize, strategy_name
from pf1e_simulator.strategy import StrategySpec

if TYPE_CHECKING:
    from pf1e_simulator.search import PlaybookEntry

ROOT = Path(__file__).resolve().parents[1]
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
ORC = ROOT / "data" / "monsters" / "orc.json"

RUNS = 5
SEED = 7
ROUND_CAP = 20
PARTY = "players"


def _scenario() -> ScenarioSpec:
    return ScenarioSpec(
        name="goblins-vs-orcs",
        map_file=MAP,
        sides=(
            SideSpec(
                name="players",
                zone="A",
                combatant_files=(GOBLIN, GOBLIN, GOBLIN),
            ),
            SideSpec(
                name="monsters",
                zone="B",
                combatant_files=(ORC, ORC),
            ),
        ),
        round_cap=ROUND_CAP,
    )


@pytest.fixture(scope="module")
def playbook() -> tuple[PlaybookEntry, ...]:
    return optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)


def test_evaluate_default_strategy() -> None:
    """evaluate returns a benchmark with sane danger metrics and win band."""
    benchmark = evaluate(
        StrategySpec(),
        _scenario(),
        party=PARTY,
        runs=100,
        seed=SEED,
    )

    assert benchmark.scenario_name == "goblins-vs-orcs"
    assert benchmark.strategy == StrategySpec()
    assert benchmark.danger.runs == 100
    assert 0.0 <= benchmark.danger.p_survives <= 1.0
    assert 0.0 <= benchmark.danger.p_tpk <= 1.0
    assert benchmark.danger.expected_rounds > 0.0
    assert set(benchmark.wins) == {"players", "monsters"}
    assert benchmark.draws >= 0
    low, high = benchmark.win_rate_band
    assert 0.0 <= low <= high <= 1.0


def test_strategy_name() -> None:
    """Labels render non-default levers in field order; the name field is excluded."""
    assert strategy_name(StrategySpec()) == "default"
    assert strategy_name(StrategySpec(target_priority="weakest_ac")) == "weakest_ac"
    assert (
        strategy_name(StrategySpec(target_priority="weakest_ac", spell_policy="none"))
        == "weakest_ac, spell_policy=none"
    )
    assert (
        strategy_name(
            StrategySpec(target_priority="lowest_hp", retreat_hp_fraction=0.5)
        )
        == "lowest_hp, retreat=0.5"
    )
    assert (
        strategy_name(StrategySpec(name="custom", use_charges=False))
        == "use_charges=false"
    )


def test_optimize_small_ranks_and_default(playbook: tuple[PlaybookEntry, ...]) -> None:
    """A small optimize returns contiguous ranks ordered by the rank key."""
    assert len(playbook) >= 24
    assert [entry.rank for entry in playbook] == list(range(1, len(playbook) + 1))
    assert any(entry.strategy == StrategySpec() for entry in playbook)

    keys = [
        (
            -entry.benchmark.danger.p_survives,
            entry.benchmark.danger.expected_hp_lost,
            -entry.benchmark.danger.worst_decile_party_hp,
            entry.benchmark.danger.expected_rounds,
            strategy_name(entry.strategy),
        )
        for entry in playbook
    ]
    assert keys == sorted(keys)


def test_optimize_deterministic() -> None:
    """Two optimize calls with the same inputs return identical playbooks."""
    first = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
    second = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
    assert first == second


def test_optimize_candidates_deduped() -> None:
    """LLM candidates run once each; duplicates and grid duplicates collapse."""
    in_grid = StrategySpec(target_priority="weakest_ac", spell_policy="none")
    custom = StrategySpec(
        target_priority="lowest_hp",
        spell_policy="none",
        use_charges=False,
        use_class_abilities=False,
        retreat_hp_fraction=0.75,
    )
    playbook = optimize(
        _scenario(),
        party=PARTY,
        runs=RUNS,
        seed=SEED,
        candidates=(custom, custom, in_grid),
    )

    custom_matches = [entry for entry in playbook if entry.strategy == custom]
    assert len(custom_matches) == 1
    in_grid_matches = [entry for entry in playbook if entry.strategy == in_grid]
    assert len(in_grid_matches) == 1

    direct = evaluate(custom, _scenario(), party=PARTY, runs=RUNS, seed=SEED)
    assert custom_matches[0].benchmark.danger == direct.danger


def test_budget_raises_below_required_total() -> None:
    """A max_total_runs below the required total raises before running."""
    with pytest.raises(ValueError, match="budget") as excinfo:
        optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=1)
    assert "max_total_runs" in str(excinfo.value)


def test_budget_generous_cap_raises_nothing() -> None:
    """A generous max_total_runs cap does not raise."""
    playbook = optimize(
        _scenario(),
        party=PARTY,
        runs=RUNS,
        seed=SEED,
        max_total_runs=10_000,
    )
    assert len(playbook) >= 24


def test_format_playbook_columns_and_ascii(
    playbook: tuple[PlaybookEntry, ...],
) -> None:
    """The renderer shows header tokens, a dash separator, aligned rows, ASCII."""
    text = format_playbook(playbook, PARTY)

    for token in (
        "Rank",
        "Strategy",
        "Runs",
        "P(survive)",
        "P(TPK)",
        "P(>=1 death)",
        "HP lost",
        "Worst-decile HP",
        "Rounds",
        "Win% (3σ)",
    ):
        assert token in text

    lines = text.splitlines()
    assert len(lines) == 2 + len(playbook)
    separator = lines[1]
    assert separator
    assert set(separator) <= {"-", " "}
    assert len({len(line) for line in lines}) == 1

    non_ascii = {char for char in text if ord(char) > 127}
    assert non_ascii <= {"σ"}

    top = playbook[0]
    rate = top.benchmark.wins.get(PARTY, 0) / top.benchmark.danger.runs
    low, high = top.benchmark.win_rate_band
    expected = f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]"
    assert expected in text