600 lines
18 KiB
Python
600 lines
18 KiB
Python
"""Tests for the robustness layer (P4.2/P4.3, docs section 10)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import itertools
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from pf1e_simulator import robustness
|
|
from pf1e_simulator.adversary import AdversarySpec
|
|
from pf1e_simulator.robustness import (
|
|
DangerCell,
|
|
MatchupBenchmark,
|
|
RobustnessEntry,
|
|
SensitivityEntry,
|
|
adversarial_analysis,
|
|
danger_map,
|
|
format_danger_map,
|
|
format_frontier,
|
|
format_matchup,
|
|
format_robustness,
|
|
format_sensitivity,
|
|
frontier,
|
|
matchup_benchmark,
|
|
sensitivity,
|
|
)
|
|
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
|
|
from pf1e_simulator.strategy import StrategySpec
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
RUNS = 5
|
|
SEED = 7
|
|
ROUND_CAP = 20
|
|
PARTY = "players"
|
|
|
|
ARENA = ROOT / "data" / "maps" / "sample_arena.yaml"
|
|
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
|
ORC = ROOT / "data" / "monsters" / "orc.json"
|
|
|
|
|
|
def _scenario(
|
|
*, name: str = "goblins-vs-orcs", sides: tuple[SideSpec, ...] | None = None
|
|
) -> ScenarioSpec:
|
|
return ScenarioSpec(
|
|
name=name,
|
|
map_file=ARENA,
|
|
sides=sides
|
|
or (
|
|
SideSpec(PARTY, "A", (GOBLIN, GOBLIN, GOBLIN)),
|
|
SideSpec("monsters", "B", (ORC, ORC)),
|
|
),
|
|
round_cap=ROUND_CAP,
|
|
)
|
|
|
|
|
|
def _party(*, name: str = "default") -> StrategySpec:
|
|
return StrategySpec(name=name)
|
|
|
|
|
|
def _three_side_scenario() -> ScenarioSpec:
|
|
return ScenarioSpec(
|
|
name="three-sides",
|
|
map_file=ARENA,
|
|
sides=(
|
|
SideSpec(PARTY, "A", (GOBLIN,)),
|
|
SideSpec("monsters", "B", (ORC,)),
|
|
SideSpec("spectators", "C", (GOBLIN,)),
|
|
),
|
|
round_cap=ROUND_CAP,
|
|
)
|
|
|
|
|
|
def test_matchup_requires_exactly_two_sides() -> None:
|
|
with pytest.raises(ValueError, match="exactly 2 sides"):
|
|
matchup_benchmark(
|
|
_three_side_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
|
|
)
|
|
|
|
|
|
def test_matchup_requires_valid_party_side() -> None:
|
|
scenario = _scenario()
|
|
with pytest.raises(ValueError, match="party_side"):
|
|
matchup_benchmark(
|
|
scenario, _party(), AdversarySpec(), party_side="", runs=RUNS, seed=SEED
|
|
)
|
|
with pytest.raises(ValueError, match="is not a side"):
|
|
matchup_benchmark(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
party_side="spectators",
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
|
|
|
|
def test_matchup_reports_both_dangers_and_is_reproducible() -> None:
|
|
scenario = _scenario()
|
|
party = _party()
|
|
adversary = AdversarySpec()
|
|
first = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
|
|
assert isinstance(first, MatchupBenchmark)
|
|
assert first.scenario == "goblins-vs-orcs"
|
|
assert first.party == "default"
|
|
assert first.adversary == "default"
|
|
assert first.runs == RUNS
|
|
assert first.seed == SEED
|
|
assert first.party_side == PARTY
|
|
assert first.report.runs == RUNS
|
|
for danger in (first.party_danger, first.adversary_danger):
|
|
assert 0.0 <= danger.p_survives <= 1.0
|
|
assert 0.0 <= danger.p_tpk <= 1.0
|
|
assert 0.0 <= danger.p_any_death <= 1.0
|
|
assert danger.expected_hp_lost >= 0.0
|
|
second = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
|
|
assert first == second
|
|
assert second.party_danger == first.party_danger
|
|
|
|
|
|
def test_adversarial_analysis_alternates_and_converges() -> None:
|
|
scenario = _scenario()
|
|
party = _party()
|
|
specs = (
|
|
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
|
|
AdversarySpec(name="plain"),
|
|
)
|
|
entries = adversarial_analysis(
|
|
scenario,
|
|
party_candidates=[party],
|
|
adversary_candidates=specs,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
iterations=2,
|
|
)
|
|
assert len(entries) == 2
|
|
assert all(isinstance(entry, RobustnessEntry) for entry in entries)
|
|
assert [entry.iteration for entry in entries] == [1, 2]
|
|
# the adversary best response is the candidate with the worst party outlook
|
|
benchmarks = {
|
|
spec.name: matchup_benchmark(
|
|
scenario, party, spec, runs=RUNS, seed=SEED
|
|
)
|
|
for spec in specs
|
|
}
|
|
expected = min(
|
|
benchmarks.values(),
|
|
key=lambda b: (
|
|
b.party_danger.p_survives,
|
|
-b.party_danger.expected_hp_lost,
|
|
b.adversary,
|
|
),
|
|
)
|
|
assert all(entry.benchmark == expected for entry in entries)
|
|
assert entries[0].adversary.name == expected.adversary
|
|
|
|
|
|
def test_adversarial_party_best_response() -> None:
|
|
scenario = _scenario()
|
|
parties = (
|
|
StrategySpec(),
|
|
StrategySpec(name="bully", target_priority="lowest_hp"),
|
|
)
|
|
adversary = AdversarySpec()
|
|
entries = adversarial_analysis(
|
|
scenario,
|
|
party_candidates=parties,
|
|
adversary_candidates=[adversary],
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
iterations=1,
|
|
)
|
|
benchmarks = {
|
|
spec.name: matchup_benchmark(
|
|
scenario, spec, adversary, runs=RUNS, seed=SEED
|
|
)
|
|
for spec in parties
|
|
}
|
|
expected = max(
|
|
benchmarks.values(),
|
|
key=lambda b: (
|
|
b.party_danger.p_survives,
|
|
-b.party_danger.expected_hp_lost,
|
|
b.party,
|
|
),
|
|
)
|
|
assert entries[0].benchmark == expected
|
|
|
|
|
|
def test_adversarial_candidates_required() -> None:
|
|
scenario = _scenario()
|
|
party = _party()
|
|
with pytest.raises(ValueError, match="party_candidates must be non-empty"):
|
|
adversarial_analysis(
|
|
scenario,
|
|
party_candidates=[],
|
|
adversary_candidates=[AdversarySpec()],
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="adversary_candidates must be non-empty"):
|
|
adversarial_analysis(
|
|
scenario,
|
|
party_candidates=[party],
|
|
adversary_candidates=[],
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
|
|
|
|
def test_adversarial_budget_is_enforced_before_any_battle() -> None:
|
|
scenario = _scenario()
|
|
party = _party()
|
|
specs = (
|
|
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
|
|
AdversarySpec(name="plain"),
|
|
)
|
|
cost = 2 * (1 + len(specs)) * RUNS
|
|
with pytest.raises(ValueError, match="robustness budget exceeded"):
|
|
adversarial_analysis(
|
|
scenario,
|
|
party_candidates=[party],
|
|
adversary_candidates=specs,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
iterations=2,
|
|
max_total_runs=cost - 1,
|
|
)
|
|
entries = adversarial_analysis(
|
|
scenario,
|
|
party_candidates=[party],
|
|
adversary_candidates=specs,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
iterations=2,
|
|
max_total_runs=cost,
|
|
)
|
|
assert len(entries) == 2
|
|
|
|
|
|
def test_danger_map_is_3x3_party_major_with_variant_names() -> None:
|
|
scenario = _scenario()
|
|
cells = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
|
assert len(cells) == 9
|
|
assert all(isinstance(cell, DangerCell) for cell in cells)
|
|
order = [(cell.party_hp_fraction, cell.adversary_hp_fraction) for cell in cells]
|
|
assert order == [(p, a) for p in (0.5, 0.7, 1.0) for a in (0.5, 0.7, 1.0)]
|
|
assert cells[0].benchmark.scenario == "goblins-vs-orcs party_hp=0.5 adv_hp=0.5"
|
|
assert cells[-1].benchmark.scenario == "goblins-vs-orcs party_hp=1.0 adv_hp=1.0"
|
|
for cell in cells:
|
|
assert 0.0 <= cell.benchmark.party_danger.p_survives <= 1.0
|
|
|
|
|
|
def test_danger_map_rejects_out_of_range_fractions() -> None:
|
|
scenario = _scenario()
|
|
with pytest.raises(ValueError, match="fraction must be in"):
|
|
danger_map(
|
|
scenario, _party(), AdversarySpec(), fractions=(0.0,), runs=RUNS, seed=SEED
|
|
)
|
|
with pytest.raises(ValueError, match="fraction must be in"):
|
|
danger_map(
|
|
scenario, _party(), AdversarySpec(), fractions=(1.5,), runs=RUNS, seed=SEED
|
|
)
|
|
with pytest.raises(ValueError, match="fractions must be non-empty"):
|
|
danger_map(
|
|
scenario, _party(), AdversarySpec(), fractions=(), runs=RUNS, seed=SEED
|
|
)
|
|
|
|
|
|
def test_danger_map_is_reproducible() -> None:
|
|
scenario = _scenario()
|
|
first = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
|
second = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
|
assert first == second
|
|
|
|
|
|
def test_frontier_finds_reached_threshold() -> None:
|
|
scenario = _scenario()
|
|
result = frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
min_fraction=0.25,
|
|
max_fraction=1.0,
|
|
precision=0.05,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
# expected hp lost is always >= 0, so the threshold is crossed everywhere:
|
|
# bisection converges to the smallest probed fraction, rounded to precision
|
|
assert result == pytest.approx(0.25)
|
|
|
|
|
|
def test_frontier_returns_none_when_threshold_unreachable() -> None:
|
|
scenario = _scenario()
|
|
result = frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=1e9,
|
|
min_fraction=0.25,
|
|
max_fraction=1.0,
|
|
precision=0.05,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
assert result is None
|
|
|
|
|
|
def test_frontier_is_reproducible() -> None:
|
|
scenario = _scenario()
|
|
first = frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
second = frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
assert first == second
|
|
|
|
|
|
def test_frontier_returns_precision_multiple() -> None:
|
|
scenario = _scenario()
|
|
result = frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
min_fraction=0.3,
|
|
max_fraction=1.0,
|
|
precision=0.1,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
assert result is not None
|
|
# bisection stops within precision of min_fraction; the result is the
|
|
# nearest precision multiple in [min_fraction, min_fraction + precision]
|
|
assert 0.3 <= result <= 0.4
|
|
assert result == pytest.approx(round(result / 0.1) * 0.1)
|
|
|
|
|
|
def test_frontier_p_survives_uses_inverse_threshold() -> None:
|
|
scenario = _scenario()
|
|
result = frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="p_survives",
|
|
target=0.9999,
|
|
min_fraction=0.5,
|
|
max_fraction=1.0,
|
|
precision=0.25,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
# convention: p_survives crosses when value <= target; a present result
|
|
# is a precision multiple within the probed range
|
|
if result is not None:
|
|
assert 0.5 <= result <= 0.75
|
|
assert result == pytest.approx(round(result / 0.25) * 0.25)
|
|
|
|
|
|
def test_frontier_validates_arguments() -> None:
|
|
scenario = _scenario()
|
|
with pytest.raises(ValueError, match="criterion must be one of"):
|
|
frontier(
|
|
scenario, _party(), AdversarySpec(), criterion="bogus", runs=RUNS, seed=SEED
|
|
)
|
|
with pytest.raises(ValueError, match="target must be in"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="p_tpk",
|
|
target=0.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="target must be in"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="p_survives",
|
|
target=1.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="target must be >= 0"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=-1.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="min_fraction must be > 0"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
min_fraction=0.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="max_fraction must be <= 1"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
max_fraction=1.5,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="min_fraction must be <= max_fraction"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
min_fraction=0.9,
|
|
max_fraction=0.5,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
with pytest.raises(ValueError, match="precision must be > 0"):
|
|
frontier(
|
|
scenario,
|
|
_party(),
|
|
AdversarySpec(),
|
|
criterion="expected_hp_lost",
|
|
target=0.0,
|
|
precision=0.0,
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
)
|
|
|
|
|
|
def test_sensitivity_ranks_levers_and_reports_signed_deltas() -> None:
|
|
scenario = _scenario()
|
|
party = _party()
|
|
entries = sensitivity(scenario, party, runs=RUNS, seed=SEED)
|
|
assert len(entries) == 7
|
|
assert all(isinstance(entry, SensitivityEntry) for entry in entries)
|
|
assert {entry.levers for entry in entries} == {
|
|
"weakest_ac",
|
|
"lowest_hp",
|
|
"healer",
|
|
"spell_policy=none",
|
|
"retreat=0.25",
|
|
"retreat=0.5",
|
|
"use_charges=false",
|
|
}
|
|
for previous, current in itertools.pairwise(entries):
|
|
assert abs(previous.delta) >= abs(current.delta)
|
|
if abs(previous.delta) == abs(current.delta):
|
|
assert previous.levers <= current.levers
|
|
# delta is the signed change vs the all-default baseline
|
|
baseline = matchup_benchmark(
|
|
scenario, party, AdversarySpec(), runs=RUNS, seed=SEED
|
|
)
|
|
base = baseline.party_danger.expected_hp_lost
|
|
for entry in entries:
|
|
assert entry.delta == pytest.approx(
|
|
entry.benchmark.party_danger.expected_hp_lost - base
|
|
)
|
|
|
|
|
|
def test_sensitivity_rejects_unknown_criterion() -> None:
|
|
scenario = _scenario()
|
|
with pytest.raises(ValueError, match="criterion must be one of"):
|
|
sensitivity(scenario, _party(), criterion="bogus", runs=RUNS, seed=SEED)
|
|
|
|
|
|
def test_sensitivity_budget_is_enforced() -> None:
|
|
scenario = _scenario()
|
|
party = _party()
|
|
with pytest.raises(ValueError, match="robustness budget exceeded"):
|
|
sensitivity(
|
|
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS - 1
|
|
)
|
|
entries = sensitivity(
|
|
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS
|
|
)
|
|
assert len(entries) == 7
|
|
|
|
|
|
def test_format_matchup_ascii_aligned() -> None:
|
|
benchmark_ = matchup_benchmark(
|
|
_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
|
|
)
|
|
text = format_matchup(benchmark_)
|
|
lines = text.splitlines()
|
|
assert lines[0].split() == [
|
|
"Scenario",
|
|
"Runs",
|
|
"Side",
|
|
"Win%",
|
|
"Survive",
|
|
"TPK",
|
|
">=1-death",
|
|
"HP-lost",
|
|
"Rounds",
|
|
]
|
|
assert lines[1] == "-" * len(lines[0])
|
|
assert "party" in lines[2]
|
|
assert "adversary" in lines[3]
|
|
assert all(len(line) == len(lines[0]) for line in lines)
|
|
assert text.isascii()
|
|
|
|
|
|
def test_format_robustness_ascii_aligned() -> None:
|
|
scenario = _scenario()
|
|
entries = adversarial_analysis(
|
|
scenario,
|
|
party_candidates=[_party()],
|
|
adversary_candidates=[AdversarySpec()],
|
|
runs=RUNS,
|
|
seed=SEED,
|
|
iterations=2,
|
|
)
|
|
text = format_robustness(entries)
|
|
lines = text.splitlines()
|
|
assert lines[0].split() == [
|
|
"Iter",
|
|
"Party",
|
|
"Adversary",
|
|
"P(party-survive)",
|
|
"P(party-TPK)",
|
|
"Party-HP",
|
|
"P(adv-survive)",
|
|
"P(adv-TPK)",
|
|
"Adv-HP",
|
|
]
|
|
assert len(lines) == 2 + 2
|
|
assert all(len(line) == len(lines[0]) for line in lines)
|
|
assert text.isascii()
|
|
|
|
|
|
def test_format_danger_map_ascii_aligned() -> None:
|
|
cells = danger_map(_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
|
text = format_danger_map(cells)
|
|
lines = text.splitlines()
|
|
assert lines[0].startswith("Party\\Adv")
|
|
assert "adv=0.5" in lines[0]
|
|
assert "party=0.5" in lines[2]
|
|
assert "party=1.0" in lines[4]
|
|
assert "/" in lines[2]
|
|
assert all(len(line) == len(lines[0]) for line in lines)
|
|
assert text.isascii()
|
|
|
|
|
|
def test_format_frontier_messages() -> None:
|
|
assert (
|
|
format_frontier("p_tpk", 0.2, None, min_fraction=0.25, max_fraction=1.0)
|
|
== "no frontier for p_tpk 0.2 in [0.25, 1.00]"
|
|
)
|
|
assert (
|
|
format_frontier("p_tpk", 0.2, 0.35, min_fraction=0.25, max_fraction=1.0)
|
|
== "frontier p_tpk 0.2: party hp_fraction = 0.35"
|
|
)
|
|
|
|
|
|
def test_format_sensitivity_ascii_aligned() -> None:
|
|
entries = sensitivity(_scenario(), _party(), runs=RUNS, seed=SEED)
|
|
text = format_sensitivity(entries, "expected_hp_lost")
|
|
lines = text.splitlines()
|
|
assert lines[0].split() == ["Rank", "Levers", "Value", "Delta"]
|
|
assert len(lines) == 2 + 7
|
|
assert lines[1].split()[-1].startswith(("+", "-"))
|
|
assert all(len(line) == len(lines[0]) for line in lines)
|
|
assert text.isascii()
|
|
|
|
|
|
def test_robustness_module_does_not_import_search() -> None:
|
|
source = Path(robustness.__file__).resolve().read_text(encoding="utf-8")
|
|
assert "search" not in source
|