ctan/pf1e-simulator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(phase2): adversary and robustness cartography (P4)

Browse Source
This commit is contained in:
Thien An
2026-09-25 21:34:11 +02:00
parent 016d6e1459
commit cd69312ff4
7 changed files with 1692 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_phase2_robustness.py
+599
View File
@@ -0,0 +1,599 @@
"""Tests for the robustness layer (P4.2/P4.3, docs section 10)."""
from __future__ import annotations
import itertools
from pathlib import Path
import pytest
from pf1e_simulator import robustness
from pf1e_simulator.adversary import AdversarySpec
from pf1e_simulator.robustness import (
DangerCell,
MatchupBenchmark,
RobustnessEntry,
SensitivityEntry,
adversarial_analysis,
danger_map,
format_danger_map,
format_frontier,
format_matchup,
format_robustness,
format_sensitivity,
frontier,
matchup_benchmark,
sensitivity,
)
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
from pf1e_simulator.strategy import StrategySpec
ROOT = Path(__file__).resolve().parent.parent
RUNS = 5
SEED = 7
ROUND_CAP = 20
PARTY = "players"
ARENA = ROOT / "data" / "maps" / "sample_arena.yaml"
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
ORC = ROOT / "data" / "monsters" / "orc.json"
def _scenario(
*, name: str = "goblins-vs-orcs", sides: tuple[SideSpec, ...] | None = None
) -> ScenarioSpec:
return ScenarioSpec(
name=name,
map_file=ARENA,
sides=sides
or (
SideSpec(PARTY, "A", (GOBLIN, GOBLIN, GOBLIN)),
SideSpec("monsters", "B", (ORC, ORC)),
),
round_cap=ROUND_CAP,
)
def _party(*, name: str = "default") -> StrategySpec:
return StrategySpec(name=name)
def _three_side_scenario() -> ScenarioSpec:
return ScenarioSpec(
name="three-sides",
map_file=ARENA,
sides=(
SideSpec(PARTY, "A", (GOBLIN,)),
SideSpec("monsters", "B", (ORC,)),
SideSpec("spectators", "C", (GOBLIN,)),
),
round_cap=ROUND_CAP,
)
def test_matchup_requires_exactly_two_sides() -> None:
with pytest.raises(ValueError, match="exactly 2 sides"):
matchup_benchmark(
_three_side_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
)
def test_matchup_requires_valid_party_side() -> None:
scenario = _scenario()
with pytest.raises(ValueError, match="party_side"):
matchup_benchmark(
scenario, _party(), AdversarySpec(), party_side="", runs=RUNS, seed=SEED
)
with pytest.raises(ValueError, match="is not a side"):
matchup_benchmark(
scenario,
_party(),
AdversarySpec(),
party_side="spectators",
runs=RUNS,
seed=SEED,
)
def test_matchup_reports_both_dangers_and_is_reproducible() -> None:
scenario = _scenario()
party = _party()
adversary = AdversarySpec()
first = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
assert isinstance(first, MatchupBenchmark)
assert first.scenario == "goblins-vs-orcs"
assert first.party == "default"
assert first.adversary == "default"
assert first.runs == RUNS
assert first.seed == SEED
assert first.party_side == PARTY
assert first.report.runs == RUNS
for danger in (first.party_danger, first.adversary_danger):
assert 0.0 <= danger.p_survives <= 1.0
assert 0.0 <= danger.p_tpk <= 1.0
assert 0.0 <= danger.p_any_death <= 1.0
assert danger.expected_hp_lost >= 0.0
second = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
assert first == second
assert second.party_danger == first.party_danger
def test_adversarial_analysis_alternates_and_converges() -> None:
scenario = _scenario()
party = _party()
specs = (
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
AdversarySpec(name="plain"),
)
entries = adversarial_analysis(
scenario,
party_candidates=[party],
adversary_candidates=specs,
runs=RUNS,
seed=SEED,
iterations=2,
)
assert len(entries) == 2
assert all(isinstance(entry, RobustnessEntry) for entry in entries)
assert [entry.iteration for entry in entries] == [1, 2]
# the adversary best response is the candidate with the worst party outlook
benchmarks = {
spec.name: matchup_benchmark(
scenario, party, spec, runs=RUNS, seed=SEED
)
for spec in specs
}
expected = min(
benchmarks.values(),
key=lambda b: (
b.party_danger.p_survives,
-b.party_danger.expected_hp_lost,
b.adversary,
),
)
assert all(entry.benchmark == expected for entry in entries)
assert entries[0].adversary.name == expected.adversary
def test_adversarial_party_best_response() -> None:
scenario = _scenario()
parties = (
StrategySpec(),
StrategySpec(name="bully", target_priority="lowest_hp"),
)
adversary = AdversarySpec()
entries = adversarial_analysis(
scenario,
party_candidates=parties,
adversary_candidates=[adversary],
runs=RUNS,
seed=SEED,
iterations=1,
)
benchmarks = {
spec.name: matchup_benchmark(
scenario, spec, adversary, runs=RUNS, seed=SEED
)
for spec in parties
}
expected = max(
benchmarks.values(),
key=lambda b: (
b.party_danger.p_survives,
-b.party_danger.expected_hp_lost,
b.party,
),
)
assert entries[0].benchmark == expected
def test_adversarial_candidates_required() -> None:
scenario = _scenario()
party = _party()
with pytest.raises(ValueError, match="party_candidates must be non-empty"):
adversarial_analysis(
scenario,
party_candidates=[],
adversary_candidates=[AdversarySpec()],
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="adversary_candidates must be non-empty"):
adversarial_analysis(
scenario,
party_candidates=[party],
adversary_candidates=[],
runs=RUNS,
seed=SEED,
)
def test_adversarial_budget_is_enforced_before_any_battle() -> None:
scenario = _scenario()
party = _party()
specs = (
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
AdversarySpec(name="plain"),
)
cost = 2 * (1 + len(specs)) * RUNS
with pytest.raises(ValueError, match="robustness budget exceeded"):
adversarial_analysis(
scenario,
party_candidates=[party],
adversary_candidates=specs,
runs=RUNS,
seed=SEED,
iterations=2,
max_total_runs=cost - 1,
)
entries = adversarial_analysis(
scenario,
party_candidates=[party],
adversary_candidates=specs,
runs=RUNS,
seed=SEED,
iterations=2,
max_total_runs=cost,
)
assert len(entries) == 2
def test_danger_map_is_3x3_party_major_with_variant_names() -> None:
scenario = _scenario()
cells = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
assert len(cells) == 9
assert all(isinstance(cell, DangerCell) for cell in cells)
order = [(cell.party_hp_fraction, cell.adversary_hp_fraction) for cell in cells]
assert order == [(p, a) for p in (0.5, 0.7, 1.0) for a in (0.5, 0.7, 1.0)]
assert cells[0].benchmark.scenario == "goblins-vs-orcs party_hp=0.5 adv_hp=0.5"
assert cells[-1].benchmark.scenario == "goblins-vs-orcs party_hp=1.0 adv_hp=1.0"
for cell in cells:
assert 0.0 <= cell.benchmark.party_danger.p_survives <= 1.0
def test_danger_map_rejects_out_of_range_fractions() -> None:
scenario = _scenario()
with pytest.raises(ValueError, match="fraction must be in"):
danger_map(
scenario, _party(), AdversarySpec(), fractions=(0.0,), runs=RUNS, seed=SEED
)
with pytest.raises(ValueError, match="fraction must be in"):
danger_map(
scenario, _party(), AdversarySpec(), fractions=(1.5,), runs=RUNS, seed=SEED
)
with pytest.raises(ValueError, match="fractions must be non-empty"):
danger_map(
scenario, _party(), AdversarySpec(), fractions=(), runs=RUNS, seed=SEED
)
def test_danger_map_is_reproducible() -> None:
scenario = _scenario()
first = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
second = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
assert first == second
def test_frontier_finds_reached_threshold() -> None:
scenario = _scenario()
result = frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
min_fraction=0.25,
max_fraction=1.0,
precision=0.05,
runs=RUNS,
seed=SEED,
)
# expected hp lost is always >= 0, so the threshold is crossed everywhere:
# bisection converges to the smallest probed fraction, rounded to precision
assert result == pytest.approx(0.25)
def test_frontier_returns_none_when_threshold_unreachable() -> None:
scenario = _scenario()
result = frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=1e9,
min_fraction=0.25,
max_fraction=1.0,
precision=0.05,
runs=RUNS,
seed=SEED,
)
assert result is None
def test_frontier_is_reproducible() -> None:
scenario = _scenario()
first = frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
runs=RUNS,
seed=SEED,
)
second = frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
runs=RUNS,
seed=SEED,
)
assert first == second
def test_frontier_returns_precision_multiple() -> None:
scenario = _scenario()
result = frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
min_fraction=0.3,
max_fraction=1.0,
precision=0.1,
runs=RUNS,
seed=SEED,
)
assert result is not None
# bisection stops within precision of min_fraction; the result is the
# nearest precision multiple in [min_fraction, min_fraction + precision]
assert 0.3 <= result <= 0.4
assert result == pytest.approx(round(result / 0.1) * 0.1)
def test_frontier_p_survives_uses_inverse_threshold() -> None:
scenario = _scenario()
result = frontier(
scenario,
_party(),
AdversarySpec(),
criterion="p_survives",
target=0.9999,
min_fraction=0.5,
max_fraction=1.0,
precision=0.25,
runs=RUNS,
seed=SEED,
)
# convention: p_survives crosses when value <= target; a present result
# is a precision multiple within the probed range
if result is not None:
assert 0.5 <= result <= 0.75
assert result == pytest.approx(round(result / 0.25) * 0.25)
def test_frontier_validates_arguments() -> None:
scenario = _scenario()
with pytest.raises(ValueError, match="criterion must be one of"):
frontier(
scenario, _party(), AdversarySpec(), criterion="bogus", runs=RUNS, seed=SEED
)
with pytest.raises(ValueError, match="target must be in"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="p_tpk",
target=0.0,
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="target must be in"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="p_survives",
target=1.0,
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="target must be >= 0"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=-1.0,
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="min_fraction must be > 0"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
min_fraction=0.0,
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="max_fraction must be <= 1"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
max_fraction=1.5,
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="min_fraction must be <= max_fraction"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
min_fraction=0.9,
max_fraction=0.5,
runs=RUNS,
seed=SEED,
)
with pytest.raises(ValueError, match="precision must be > 0"):
frontier(
scenario,
_party(),
AdversarySpec(),
criterion="expected_hp_lost",
target=0.0,
precision=0.0,
runs=RUNS,
seed=SEED,
)
def test_sensitivity_ranks_levers_and_reports_signed_deltas() -> None:
scenario = _scenario()
party = _party()
entries = sensitivity(scenario, party, runs=RUNS, seed=SEED)
assert len(entries) == 7
assert all(isinstance(entry, SensitivityEntry) for entry in entries)
assert {entry.levers for entry in entries} == {
"weakest_ac",
"lowest_hp",
"healer",
"spell_policy=none",
"retreat=0.25",
"retreat=0.5",
"use_charges=false",
}
for previous, current in itertools.pairwise(entries):
assert abs(previous.delta) >= abs(current.delta)
if abs(previous.delta) == abs(current.delta):
assert previous.levers <= current.levers
# delta is the signed change vs the all-default baseline
baseline = matchup_benchmark(
scenario, party, AdversarySpec(), runs=RUNS, seed=SEED
)
base = baseline.party_danger.expected_hp_lost
for entry in entries:
assert entry.delta == pytest.approx(
entry.benchmark.party_danger.expected_hp_lost - base
)
def test_sensitivity_rejects_unknown_criterion() -> None:
scenario = _scenario()
with pytest.raises(ValueError, match="criterion must be one of"):
sensitivity(scenario, _party(), criterion="bogus", runs=RUNS, seed=SEED)
def test_sensitivity_budget_is_enforced() -> None:
scenario = _scenario()
party = _party()
with pytest.raises(ValueError, match="robustness budget exceeded"):
sensitivity(
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS - 1
)
entries = sensitivity(
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS
)
assert len(entries) == 7
def test_format_matchup_ascii_aligned() -> None:
benchmark_ = matchup_benchmark(
_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
)
text = format_matchup(benchmark_)
lines = text.splitlines()
assert lines[0].split() == [
"Scenario",
"Runs",
"Side",
"Win%",
"Survive",
"TPK",
">=1-death",
"HP-lost",
"Rounds",
]
assert lines[1] == "-" * len(lines[0])
assert "party" in lines[2]
assert "adversary" in lines[3]
assert all(len(line) == len(lines[0]) for line in lines)
assert text.isascii()
def test_format_robustness_ascii_aligned() -> None:
scenario = _scenario()
entries = adversarial_analysis(
scenario,
party_candidates=[_party()],
adversary_candidates=[AdversarySpec()],
runs=RUNS,
seed=SEED,
iterations=2,
)
text = format_robustness(entries)
lines = text.splitlines()
assert lines[0].split() == [
"Iter",
"Party",
"Adversary",
"P(party-survive)",
"P(party-TPK)",
"Party-HP",
"P(adv-survive)",
"P(adv-TPK)",
"Adv-HP",
]
assert len(lines) == 2 + 2
assert all(len(line) == len(lines[0]) for line in lines)
assert text.isascii()
def test_format_danger_map_ascii_aligned() -> None:
cells = danger_map(_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED)
text = format_danger_map(cells)
lines = text.splitlines()
assert lines[0].startswith("Party\\Adv")
assert "adv=0.5" in lines[0]
assert "party=0.5" in lines[2]
assert "party=1.0" in lines[4]
assert "/" in lines[2]
assert all(len(line) == len(lines[0]) for line in lines)
assert text.isascii()
def test_format_frontier_messages() -> None:
assert (
format_frontier("p_tpk", 0.2, None, min_fraction=0.25, max_fraction=1.0)
== "no frontier for p_tpk 0.2 in [0.25, 1.00]"
)
assert (
format_frontier("p_tpk", 0.2, 0.35, min_fraction=0.25, max_fraction=1.0)
== "frontier p_tpk 0.2: party hp_fraction = 0.35"
)
def test_format_sensitivity_ascii_aligned() -> None:
entries = sensitivity(_scenario(), _party(), runs=RUNS, seed=SEED)
text = format_sensitivity(entries, "expected_hp_lost")
lines = text.splitlines()
assert lines[0].split() == ["Rank", "Levers", "Value", "Delta"]
assert len(lines) == 2 + 7
assert lines[1].split()[-1].startswith(("+", "-"))
assert all(len(line) == len(lines[0]) for line in lines)
assert text.isascii()
def test_robustness_module_does_not_import_search() -> None:
source = Path(robustness.__file__).resolve().read_text(encoding="utf-8")
assert "search" not in source