"""Tests for the robustness layer (P4.2/P4.3, docs section 10).""" from __future__ import annotations import itertools from pathlib import Path import pytest from pf1e_simulator import robustness from pf1e_simulator.adversary import AdversarySpec from pf1e_simulator.robustness import ( DangerCell, MatchupBenchmark, RobustnessEntry, SensitivityEntry, adversarial_analysis, danger_map, format_danger_map, format_frontier, format_matchup, format_robustness, format_sensitivity, frontier, matchup_benchmark, sensitivity, ) from pf1e_simulator.scenarios import ScenarioSpec, SideSpec from pf1e_simulator.strategy import StrategySpec ROOT = Path(__file__).resolve().parent.parent RUNS = 5 SEED = 7 ROUND_CAP = 20 PARTY = "players" ARENA = ROOT / "data" / "maps" / "sample_arena.yaml" GOBLIN = ROOT / "data" / "monsters" / "goblin.json" ORC = ROOT / "data" / "monsters" / "orc.json" def _scenario( *, name: str = "goblins-vs-orcs", sides: tuple[SideSpec, ...] | None = None ) -> ScenarioSpec: return ScenarioSpec( name=name, map_file=ARENA, sides=sides or ( SideSpec(PARTY, "A", (GOBLIN, GOBLIN, GOBLIN)), SideSpec("monsters", "B", (ORC, ORC)), ), round_cap=ROUND_CAP, ) def _party(*, name: str = "default") -> StrategySpec: return StrategySpec(name=name) def _three_side_scenario() -> ScenarioSpec: return ScenarioSpec( name="three-sides", map_file=ARENA, sides=( SideSpec(PARTY, "A", (GOBLIN,)), SideSpec("monsters", "B", (ORC,)), SideSpec("spectators", "C", (GOBLIN,)), ), round_cap=ROUND_CAP, ) def test_matchup_requires_exactly_two_sides() -> None: with pytest.raises(ValueError, match="exactly 2 sides"): matchup_benchmark( _three_side_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED ) def test_matchup_requires_valid_party_side() -> None: scenario = _scenario() with pytest.raises(ValueError, match="party_side"): matchup_benchmark( scenario, _party(), AdversarySpec(), party_side="", runs=RUNS, seed=SEED ) with pytest.raises(ValueError, match="is not a side"): matchup_benchmark( scenario, _party(), AdversarySpec(), party_side="spectators", runs=RUNS, seed=SEED, ) def test_matchup_reports_both_dangers_and_is_reproducible() -> None: scenario = _scenario() party = _party() adversary = AdversarySpec() first = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED) assert isinstance(first, MatchupBenchmark) assert first.scenario == "goblins-vs-orcs" assert first.party == "default" assert first.adversary == "default" assert first.runs == RUNS assert first.seed == SEED assert first.party_side == PARTY assert first.report.runs == RUNS for danger in (first.party_danger, first.adversary_danger): assert 0.0 <= danger.p_survives <= 1.0 assert 0.0 <= danger.p_tpk <= 1.0 assert 0.0 <= danger.p_any_death <= 1.0 assert danger.expected_hp_lost >= 0.0 second = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED) assert first == second assert second.party_danger == first.party_danger def test_adversarial_analysis_alternates_and_converges() -> None: scenario = _scenario() party = _party() specs = ( AdversarySpec(name="retreat", retreat_hp_fraction=0.5), AdversarySpec(name="plain"), ) entries = adversarial_analysis( scenario, party_candidates=[party], adversary_candidates=specs, runs=RUNS, seed=SEED, iterations=2, ) assert len(entries) == 2 assert all(isinstance(entry, RobustnessEntry) for entry in entries) assert [entry.iteration for entry in entries] == [1, 2] # the adversary best response is the candidate with the worst party outlook benchmarks = { spec.name: matchup_benchmark( scenario, party, spec, runs=RUNS, seed=SEED ) for spec in specs } expected = min( benchmarks.values(), key=lambda b: ( b.party_danger.p_survives, -b.party_danger.expected_hp_lost, b.adversary, ), ) assert all(entry.benchmark == expected for entry in entries) assert entries[0].adversary.name == expected.adversary def test_adversarial_party_best_response() -> None: scenario = _scenario() parties = ( StrategySpec(), StrategySpec(name="bully", target_priority="lowest_hp"), ) adversary = AdversarySpec() entries = adversarial_analysis( scenario, party_candidates=parties, adversary_candidates=[adversary], runs=RUNS, seed=SEED, iterations=1, ) benchmarks = { spec.name: matchup_benchmark( scenario, spec, adversary, runs=RUNS, seed=SEED ) for spec in parties } expected = max( benchmarks.values(), key=lambda b: ( b.party_danger.p_survives, -b.party_danger.expected_hp_lost, b.party, ), ) assert entries[0].benchmark == expected def test_adversarial_candidates_required() -> None: scenario = _scenario() party = _party() with pytest.raises(ValueError, match="party_candidates must be non-empty"): adversarial_analysis( scenario, party_candidates=[], adversary_candidates=[AdversarySpec()], runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="adversary_candidates must be non-empty"): adversarial_analysis( scenario, party_candidates=[party], adversary_candidates=[], runs=RUNS, seed=SEED, ) def test_adversarial_budget_is_enforced_before_any_battle() -> None: scenario = _scenario() party = _party() specs = ( AdversarySpec(name="retreat", retreat_hp_fraction=0.5), AdversarySpec(name="plain"), ) cost = 2 * (1 + len(specs)) * RUNS with pytest.raises(ValueError, match="robustness budget exceeded"): adversarial_analysis( scenario, party_candidates=[party], adversary_candidates=specs, runs=RUNS, seed=SEED, iterations=2, max_total_runs=cost - 1, ) entries = adversarial_analysis( scenario, party_candidates=[party], adversary_candidates=specs, runs=RUNS, seed=SEED, iterations=2, max_total_runs=cost, ) assert len(entries) == 2 def test_danger_map_is_3x3_party_major_with_variant_names() -> None: scenario = _scenario() cells = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED) assert len(cells) == 9 assert all(isinstance(cell, DangerCell) for cell in cells) order = [(cell.party_hp_fraction, cell.adversary_hp_fraction) for cell in cells] assert order == [(p, a) for p in (0.5, 0.7, 1.0) for a in (0.5, 0.7, 1.0)] assert cells[0].benchmark.scenario == "goblins-vs-orcs party_hp=0.5 adv_hp=0.5" assert cells[-1].benchmark.scenario == "goblins-vs-orcs party_hp=1.0 adv_hp=1.0" for cell in cells: assert 0.0 <= cell.benchmark.party_danger.p_survives <= 1.0 def test_danger_map_rejects_out_of_range_fractions() -> None: scenario = _scenario() with pytest.raises(ValueError, match="fraction must be in"): danger_map( scenario, _party(), AdversarySpec(), fractions=(0.0,), runs=RUNS, seed=SEED ) with pytest.raises(ValueError, match="fraction must be in"): danger_map( scenario, _party(), AdversarySpec(), fractions=(1.5,), runs=RUNS, seed=SEED ) with pytest.raises(ValueError, match="fractions must be non-empty"): danger_map( scenario, _party(), AdversarySpec(), fractions=(), runs=RUNS, seed=SEED ) def test_danger_map_is_reproducible() -> None: scenario = _scenario() first = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED) second = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED) assert first == second def test_frontier_finds_reached_threshold() -> None: scenario = _scenario() result = frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, min_fraction=0.25, max_fraction=1.0, precision=0.05, runs=RUNS, seed=SEED, ) # expected hp lost is always >= 0, so the threshold is crossed everywhere: # bisection converges to the smallest probed fraction, rounded to precision assert result == pytest.approx(0.25) def test_frontier_returns_none_when_threshold_unreachable() -> None: scenario = _scenario() result = frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=1e9, min_fraction=0.25, max_fraction=1.0, precision=0.05, runs=RUNS, seed=SEED, ) assert result is None def test_frontier_is_reproducible() -> None: scenario = _scenario() first = frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, runs=RUNS, seed=SEED, ) second = frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, runs=RUNS, seed=SEED, ) assert first == second def test_frontier_returns_precision_multiple() -> None: scenario = _scenario() result = frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, min_fraction=0.3, max_fraction=1.0, precision=0.1, runs=RUNS, seed=SEED, ) assert result is not None # bisection stops within precision of min_fraction; the result is the # nearest precision multiple in [min_fraction, min_fraction + precision] assert 0.3 <= result <= 0.4 assert result == pytest.approx(round(result / 0.1) * 0.1) def test_frontier_p_survives_uses_inverse_threshold() -> None: scenario = _scenario() result = frontier( scenario, _party(), AdversarySpec(), criterion="p_survives", target=0.9999, min_fraction=0.5, max_fraction=1.0, precision=0.25, runs=RUNS, seed=SEED, ) # convention: p_survives crosses when value <= target; a present result # is a precision multiple within the probed range if result is not None: assert 0.5 <= result <= 0.75 assert result == pytest.approx(round(result / 0.25) * 0.25) def test_frontier_validates_arguments() -> None: scenario = _scenario() with pytest.raises(ValueError, match="criterion must be one of"): frontier( scenario, _party(), AdversarySpec(), criterion="bogus", runs=RUNS, seed=SEED ) with pytest.raises(ValueError, match="target must be in"): frontier( scenario, _party(), AdversarySpec(), criterion="p_tpk", target=0.0, runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="target must be in"): frontier( scenario, _party(), AdversarySpec(), criterion="p_survives", target=1.0, runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="target must be >= 0"): frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=-1.0, runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="min_fraction must be > 0"): frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, min_fraction=0.0, runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="max_fraction must be <= 1"): frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, max_fraction=1.5, runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="min_fraction must be <= max_fraction"): frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, min_fraction=0.9, max_fraction=0.5, runs=RUNS, seed=SEED, ) with pytest.raises(ValueError, match="precision must be > 0"): frontier( scenario, _party(), AdversarySpec(), criterion="expected_hp_lost", target=0.0, precision=0.0, runs=RUNS, seed=SEED, ) def test_sensitivity_ranks_levers_and_reports_signed_deltas() -> None: scenario = _scenario() party = _party() entries = sensitivity(scenario, party, runs=RUNS, seed=SEED) assert len(entries) == 7 assert all(isinstance(entry, SensitivityEntry) for entry in entries) assert {entry.levers for entry in entries} == { "weakest_ac", "lowest_hp", "healer", "spell_policy=none", "retreat=0.25", "retreat=0.5", "use_charges=false", } for previous, current in itertools.pairwise(entries): assert abs(previous.delta) >= abs(current.delta) if abs(previous.delta) == abs(current.delta): assert previous.levers <= current.levers # delta is the signed change vs the all-default baseline baseline = matchup_benchmark( scenario, party, AdversarySpec(), runs=RUNS, seed=SEED ) base = baseline.party_danger.expected_hp_lost for entry in entries: assert entry.delta == pytest.approx( entry.benchmark.party_danger.expected_hp_lost - base ) def test_sensitivity_rejects_unknown_criterion() -> None: scenario = _scenario() with pytest.raises(ValueError, match="criterion must be one of"): sensitivity(scenario, _party(), criterion="bogus", runs=RUNS, seed=SEED) def test_sensitivity_budget_is_enforced() -> None: scenario = _scenario() party = _party() with pytest.raises(ValueError, match="robustness budget exceeded"): sensitivity( scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS - 1 ) entries = sensitivity( scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS ) assert len(entries) == 7 def test_format_matchup_ascii_aligned() -> None: benchmark_ = matchup_benchmark( _scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED ) text = format_matchup(benchmark_) lines = text.splitlines() assert lines[0].split() == [ "Scenario", "Runs", "Side", "Win%", "Survive", "TPK", ">=1-death", "HP-lost", "Rounds", ] assert lines[1] == "-" * len(lines[0]) assert "party" in lines[2] assert "adversary" in lines[3] assert all(len(line) == len(lines[0]) for line in lines) assert text.isascii() def test_format_robustness_ascii_aligned() -> None: scenario = _scenario() entries = adversarial_analysis( scenario, party_candidates=[_party()], adversary_candidates=[AdversarySpec()], runs=RUNS, seed=SEED, iterations=2, ) text = format_robustness(entries) lines = text.splitlines() assert lines[0].split() == [ "Iter", "Party", "Adversary", "P(party-survive)", "P(party-TPK)", "Party-HP", "P(adv-survive)", "P(adv-TPK)", "Adv-HP", ] assert len(lines) == 2 + 2 assert all(len(line) == len(lines[0]) for line in lines) assert text.isascii() def test_format_danger_map_ascii_aligned() -> None: cells = danger_map(_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED) text = format_danger_map(cells) lines = text.splitlines() assert lines[0].startswith("Party\\Adv") assert "adv=0.5" in lines[0] assert "party=0.5" in lines[2] assert "party=1.0" in lines[4] assert "/" in lines[2] assert all(len(line) == len(lines[0]) for line in lines) assert text.isascii() def test_format_frontier_messages() -> None: assert ( format_frontier("p_tpk", 0.2, None, min_fraction=0.25, max_fraction=1.0) == "no frontier for p_tpk 0.2 in [0.25, 1.00]" ) assert ( format_frontier("p_tpk", 0.2, 0.35, min_fraction=0.25, max_fraction=1.0) == "frontier p_tpk 0.2: party hp_fraction = 0.35" ) def test_format_sensitivity_ascii_aligned() -> None: entries = sensitivity(_scenario(), _party(), runs=RUNS, seed=SEED) text = format_sensitivity(entries, "expected_hp_lost") lines = text.splitlines() assert lines[0].split() == ["Rank", "Levers", "Value", "Delta"] assert len(lines) == 2 + 7 assert lines[1].split()[-1].startswith(("+", "-")) assert all(len(line) == len(lines[0]) for line in lines) assert text.isascii() def test_robustness_module_does_not_import_search() -> None: source = Path(robustness.__file__).resolve().read_text(encoding="utf-8") assert "search" not in source