feat(phase2): adversary and robustness cartography (P4)
This commit is contained in:
@@ -0,0 +1,249 @@
|
||||
"""Phase-2 adversary compilation tests (docs/phase-2-llm-architecture.md 10 P4.1).
|
||||
|
||||
Deterministic, offline: no network, no LLM. Pins the ``AdversarySpec`` /
|
||||
``compile_adversary`` / ``compile_matchup`` contract (default-policy identity,
|
||||
per-side routing on a real ``CombatEngine``, empty ``party_side`` rejection),
|
||||
the additive ``"healer"`` target priority (prefer the enemy caster, fall back
|
||||
to ``weakest_ac``), and ``SideSpec.aware`` landing on the right side's
|
||||
combatants plus its SRD "Surprise" round effect. Conventions follow
|
||||
tests/test_phase2_search.py: small n (runs <= 5), fixed seed, ROOT-relative
|
||||
path constants.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.adversary import (
|
||||
AdversarySpec,
|
||||
adversary_name,
|
||||
compile_adversary,
|
||||
compile_matchup,
|
||||
)
|
||||
from pf1e_simulator.combat import CombatantState, CombatEngine, default_policy
|
||||
from pf1e_simulator.grid import Grid
|
||||
from pf1e_simulator.loaders import load_combatant
|
||||
from pf1e_simulator.map import load_map
|
||||
from pf1e_simulator.rng import SeededRng
|
||||
from pf1e_simulator.runner import build_states, run_encounter
|
||||
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec, build_encounter
|
||||
from pf1e_simulator.strategy import StrategySpec, compile_strategy, select_target
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pf1e_simulator.models import Combatant
|
||||
from pf1e_simulator.runner import EncounterSpec
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
||||
ORC = ROOT / "data" / "monsters" / "orc.json"
|
||||
NAIRDA = ROOT / "data" / "pcs" / "nairda.json"
|
||||
|
||||
RUNS = 5
|
||||
SEED = 7
|
||||
ROUND_CAP = 20
|
||||
PARTY = "players"
|
||||
|
||||
_MAP_SPEC = load_map(MAP)
|
||||
|
||||
|
||||
def _c(path: Path) -> Combatant:
|
||||
return load_combatant(path)
|
||||
|
||||
|
||||
def _scenario(*, monsters_aware: bool | None = None) -> ScenarioSpec:
|
||||
"""3 goblins (zone A) vs 2 orcs (zone B), optionally with side awareness."""
|
||||
return ScenarioSpec(
|
||||
name="goblins-vs-orcs",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(
|
||||
name=PARTY,
|
||||
zone="A",
|
||||
combatant_files=(GOBLIN, GOBLIN, GOBLIN),
|
||||
),
|
||||
SideSpec(
|
||||
name="monsters",
|
||||
zone="B",
|
||||
combatant_files=(ORC, ORC),
|
||||
aware=monsters_aware,
|
||||
),
|
||||
),
|
||||
round_cap=ROUND_CAP,
|
||||
)
|
||||
|
||||
|
||||
def _encounter() -> EncounterSpec:
|
||||
return build_encounter(_scenario())
|
||||
|
||||
|
||||
def _state(
|
||||
combatant: Combatant, side: str, pos: tuple[int, int], *, hp: int | None = None
|
||||
) -> CombatantState:
|
||||
"""A bare runtime state over a loaded combatant (no battle run)."""
|
||||
return CombatantState(
|
||||
combatant=combatant,
|
||||
side=side,
|
||||
pos=pos,
|
||||
hp=combatant.hp_max if hp is None else hp,
|
||||
resources_left={},
|
||||
)
|
||||
|
||||
|
||||
def _mini_engine(states: list[CombatantState]) -> CombatEngine:
|
||||
"""A live engine over the sample arena with the given states (no battle run)."""
|
||||
return CombatEngine(SeededRng(SEED), Grid.from_spec(_MAP_SPEC), states)
|
||||
|
||||
|
||||
# --- identity ---------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compile_adversary_default_is_default_policy() -> None:
|
||||
"""The all-default AdversarySpec compiles to default_policy by identity."""
|
||||
assert compile_adversary(AdversarySpec()) is default_policy
|
||||
|
||||
|
||||
def test_compile_matchup_defaults_is_default_policy() -> None:
|
||||
"""Two all-default specs return default_policy itself, not a wrapper."""
|
||||
assert compile_matchup(StrategySpec(), AdversarySpec()) is default_policy
|
||||
|
||||
|
||||
def test_adversary_name_mirrors_strategy_name() -> None:
|
||||
"""Labels render non-default levers in field order; the name field is excluded."""
|
||||
assert adversary_name(AdversarySpec()) == "default"
|
||||
assert adversary_name(AdversarySpec(target_priority="healer")) == "healer"
|
||||
assert (
|
||||
adversary_name(AdversarySpec(name="custom", spell_policy="none"))
|
||||
== "spell_policy=none"
|
||||
)
|
||||
assert (
|
||||
adversary_name(
|
||||
AdversarySpec(target_priority="weakest_ac", retreat_hp_fraction=0.5)
|
||||
)
|
||||
== "weakest_ac, retreat=0.5"
|
||||
)
|
||||
|
||||
|
||||
# --- validation ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compile_matchup_empty_party_side_raises() -> None:
|
||||
"""An empty party_side is a ValueError (pre-assigned msg, EM101/EM102)."""
|
||||
with pytest.raises(ValueError, match="party_side"):
|
||||
compile_matchup(StrategySpec(), AdversarySpec(), party_side="")
|
||||
|
||||
|
||||
# --- routing ----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compile_matchup_routes_by_side() -> None:
|
||||
"""On a real engine, each side of the matchup delegates to its own policy."""
|
||||
party_spec = StrategySpec(name="party-focus", target_priority="weakest_ac")
|
||||
adversary_spec = AdversarySpec(name="adv-focus", target_priority="lowest_hp")
|
||||
matchup = compile_matchup(party_spec, adversary_spec)
|
||||
assert matchup is not default_policy
|
||||
|
||||
encounter = _encounter()
|
||||
states = build_states(encounter)
|
||||
engine = CombatEngine(SeededRng(SEED), Grid.from_spec(encounter.map), states)
|
||||
party_state = next(s for s in states if s.side == PARTY)
|
||||
monster_state = next(s for s in states if s.side == "monsters")
|
||||
|
||||
assert matchup(engine, party_state) == compile_strategy(party_spec)(
|
||||
engine, party_state
|
||||
)
|
||||
assert matchup(engine, monster_state) == compile_adversary(adversary_spec)(
|
||||
engine, monster_state
|
||||
)
|
||||
|
||||
|
||||
def test_matchup_and_healer_strategy_run_deterministically() -> None:
|
||||
"""A healer spec compiles and runs evaluate-style cells bit-identically."""
|
||||
healer = StrategySpec(name="healer-party", target_priority="healer")
|
||||
policy = compile_strategy(healer)
|
||||
assert policy is not default_policy
|
||||
matchup = compile_matchup(healer, AdversarySpec(target_priority="weakest_ac"))
|
||||
|
||||
spec = _encounter()
|
||||
_, first = run_encounter(spec, runs=RUNS, seed=SEED, policy=policy, collect_runs=True)
|
||||
_, again = run_encounter(spec, runs=RUNS, seed=SEED, policy=policy, collect_runs=True)
|
||||
assert first == again
|
||||
_, matchup_a = run_encounter(
|
||||
spec, runs=RUNS, seed=SEED, policy=matchup, collect_runs=True
|
||||
)
|
||||
_, matchup_b = run_encounter(
|
||||
spec, runs=RUNS, seed=SEED, policy=matchup, collect_runs=True
|
||||
)
|
||||
assert matchup_a == matchup_b
|
||||
|
||||
|
||||
# --- healer targeting -------------------------------------------------------
|
||||
|
||||
|
||||
def test_healer_prefers_the_enemy_caster() -> None:
|
||||
"""With a monster-side caster (nairda) among non-casters, healer picks her."""
|
||||
actor = _state(_c(GOBLIN), PARTY, (1, 1))
|
||||
nairda = _state(_c(NAIRDA), "monsters", (1, 8))
|
||||
orc = _state(_c(ORC), "monsters", (1, 16))
|
||||
assert nairda.combatant.spells # caster precondition (nairda.json)
|
||||
engine = _mini_engine([actor, nairda, orc])
|
||||
|
||||
assert select_target(engine, actor, "healer") is nairda
|
||||
# Discriminator: weakest_ac sees equal AC 13 and prefers the wounded orc.
|
||||
assert select_target(engine, actor, "weakest_ac") is orc
|
||||
|
||||
|
||||
def test_healer_falls_back_to_weakest_ac_without_casters() -> None:
|
||||
"""No active caster on the enemy side -> identical to weakest_ac."""
|
||||
actor = _state(_c(GOBLIN), PARTY, (1, 1))
|
||||
orc = _state(_c(ORC), "monsters", (1, 8))
|
||||
goblin_enemy = _state(_c(GOBLIN), "monsters", (1, 16))
|
||||
assert not orc.combatant.spells
|
||||
assert not goblin_enemy.combatant.spells
|
||||
engine = _mini_engine([actor, orc, goblin_enemy])
|
||||
|
||||
healer = select_target(engine, actor, "healer")
|
||||
assert healer is select_target(engine, actor, "weakest_ac")
|
||||
assert healer is orc # AC 13 < goblin AC 16
|
||||
|
||||
|
||||
# --- SideSpec.aware (SRD "Surprise") ---------------------------------------
|
||||
|
||||
|
||||
def test_side_spec_aware_applies_per_side() -> None:
|
||||
"""aware=False lands on that side's combatants only; the other side keeps True."""
|
||||
encounter = build_encounter(_scenario(monsters_aware=False))
|
||||
players, monsters = encounter.sides
|
||||
assert all(c.aware is True for c in players.combatants)
|
||||
assert all(c.aware is False for c in monsters.combatants)
|
||||
|
||||
states = build_states(encounter)
|
||||
assert all(s.combatant.aware is True for s in states if s.side == PARTY)
|
||||
assert all(s.combatant.aware is False for s in states if s.side == "monsters")
|
||||
|
||||
untouched = build_encounter(_scenario())
|
||||
assert all(c.aware is True for side in untouched.sides for c in side.combatants)
|
||||
|
||||
|
||||
def test_side_spec_aware_triggers_surprise_round() -> None:
|
||||
"""Only some aware -> a surprise round runs; the default (aware=None) has none."""
|
||||
surprise = _encounter_states_and_run(_scenario(monsters_aware=False))
|
||||
assert "surprise round" in surprise
|
||||
|
||||
default = _encounter_states_and_run(_scenario())
|
||||
assert "surprise round" not in default
|
||||
|
||||
|
||||
def _encounter_states_and_run(scenario: ScenarioSpec) -> tuple[str, ...]:
|
||||
"""Run one seeded battle of `scenario` and return its transcript."""
|
||||
encounter = build_encounter(scenario)
|
||||
engine = CombatEngine(
|
||||
SeededRng(SEED),
|
||||
Grid.from_spec(encounter.map),
|
||||
build_states(encounter),
|
||||
round_cap=ROUND_CAP,
|
||||
)
|
||||
return engine.run().transcript
|
||||
@@ -0,0 +1,599 @@
|
||||
"""Tests for the robustness layer (P4.2/P4.3, docs section 10)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator import robustness
|
||||
from pf1e_simulator.adversary import AdversarySpec
|
||||
from pf1e_simulator.robustness import (
|
||||
DangerCell,
|
||||
MatchupBenchmark,
|
||||
RobustnessEntry,
|
||||
SensitivityEntry,
|
||||
adversarial_analysis,
|
||||
danger_map,
|
||||
format_danger_map,
|
||||
format_frontier,
|
||||
format_matchup,
|
||||
format_robustness,
|
||||
format_sensitivity,
|
||||
frontier,
|
||||
matchup_benchmark,
|
||||
sensitivity,
|
||||
)
|
||||
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
|
||||
from pf1e_simulator.strategy import StrategySpec
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
RUNS = 5
|
||||
SEED = 7
|
||||
ROUND_CAP = 20
|
||||
PARTY = "players"
|
||||
|
||||
ARENA = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
||||
ORC = ROOT / "data" / "monsters" / "orc.json"
|
||||
|
||||
|
||||
def _scenario(
|
||||
*, name: str = "goblins-vs-orcs", sides: tuple[SideSpec, ...] | None = None
|
||||
) -> ScenarioSpec:
|
||||
return ScenarioSpec(
|
||||
name=name,
|
||||
map_file=ARENA,
|
||||
sides=sides
|
||||
or (
|
||||
SideSpec(PARTY, "A", (GOBLIN, GOBLIN, GOBLIN)),
|
||||
SideSpec("monsters", "B", (ORC, ORC)),
|
||||
),
|
||||
round_cap=ROUND_CAP,
|
||||
)
|
||||
|
||||
|
||||
def _party(*, name: str = "default") -> StrategySpec:
|
||||
return StrategySpec(name=name)
|
||||
|
||||
|
||||
def _three_side_scenario() -> ScenarioSpec:
|
||||
return ScenarioSpec(
|
||||
name="three-sides",
|
||||
map_file=ARENA,
|
||||
sides=(
|
||||
SideSpec(PARTY, "A", (GOBLIN,)),
|
||||
SideSpec("monsters", "B", (ORC,)),
|
||||
SideSpec("spectators", "C", (GOBLIN,)),
|
||||
),
|
||||
round_cap=ROUND_CAP,
|
||||
)
|
||||
|
||||
|
||||
def test_matchup_requires_exactly_two_sides() -> None:
|
||||
with pytest.raises(ValueError, match="exactly 2 sides"):
|
||||
matchup_benchmark(
|
||||
_three_side_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
|
||||
)
|
||||
|
||||
|
||||
def test_matchup_requires_valid_party_side() -> None:
|
||||
scenario = _scenario()
|
||||
with pytest.raises(ValueError, match="party_side"):
|
||||
matchup_benchmark(
|
||||
scenario, _party(), AdversarySpec(), party_side="", runs=RUNS, seed=SEED
|
||||
)
|
||||
with pytest.raises(ValueError, match="is not a side"):
|
||||
matchup_benchmark(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
party_side="spectators",
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
|
||||
def test_matchup_reports_both_dangers_and_is_reproducible() -> None:
|
||||
scenario = _scenario()
|
||||
party = _party()
|
||||
adversary = AdversarySpec()
|
||||
first = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
|
||||
assert isinstance(first, MatchupBenchmark)
|
||||
assert first.scenario == "goblins-vs-orcs"
|
||||
assert first.party == "default"
|
||||
assert first.adversary == "default"
|
||||
assert first.runs == RUNS
|
||||
assert first.seed == SEED
|
||||
assert first.party_side == PARTY
|
||||
assert first.report.runs == RUNS
|
||||
for danger in (first.party_danger, first.adversary_danger):
|
||||
assert 0.0 <= danger.p_survives <= 1.0
|
||||
assert 0.0 <= danger.p_tpk <= 1.0
|
||||
assert 0.0 <= danger.p_any_death <= 1.0
|
||||
assert danger.expected_hp_lost >= 0.0
|
||||
second = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
|
||||
assert first == second
|
||||
assert second.party_danger == first.party_danger
|
||||
|
||||
|
||||
def test_adversarial_analysis_alternates_and_converges() -> None:
|
||||
scenario = _scenario()
|
||||
party = _party()
|
||||
specs = (
|
||||
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
|
||||
AdversarySpec(name="plain"),
|
||||
)
|
||||
entries = adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=[party],
|
||||
adversary_candidates=specs,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
iterations=2,
|
||||
)
|
||||
assert len(entries) == 2
|
||||
assert all(isinstance(entry, RobustnessEntry) for entry in entries)
|
||||
assert [entry.iteration for entry in entries] == [1, 2]
|
||||
# the adversary best response is the candidate with the worst party outlook
|
||||
benchmarks = {
|
||||
spec.name: matchup_benchmark(
|
||||
scenario, party, spec, runs=RUNS, seed=SEED
|
||||
)
|
||||
for spec in specs
|
||||
}
|
||||
expected = min(
|
||||
benchmarks.values(),
|
||||
key=lambda b: (
|
||||
b.party_danger.p_survives,
|
||||
-b.party_danger.expected_hp_lost,
|
||||
b.adversary,
|
||||
),
|
||||
)
|
||||
assert all(entry.benchmark == expected for entry in entries)
|
||||
assert entries[0].adversary.name == expected.adversary
|
||||
|
||||
|
||||
def test_adversarial_party_best_response() -> None:
|
||||
scenario = _scenario()
|
||||
parties = (
|
||||
StrategySpec(),
|
||||
StrategySpec(name="bully", target_priority="lowest_hp"),
|
||||
)
|
||||
adversary = AdversarySpec()
|
||||
entries = adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=parties,
|
||||
adversary_candidates=[adversary],
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
iterations=1,
|
||||
)
|
||||
benchmarks = {
|
||||
spec.name: matchup_benchmark(
|
||||
scenario, spec, adversary, runs=RUNS, seed=SEED
|
||||
)
|
||||
for spec in parties
|
||||
}
|
||||
expected = max(
|
||||
benchmarks.values(),
|
||||
key=lambda b: (
|
||||
b.party_danger.p_survives,
|
||||
-b.party_danger.expected_hp_lost,
|
||||
b.party,
|
||||
),
|
||||
)
|
||||
assert entries[0].benchmark == expected
|
||||
|
||||
|
||||
def test_adversarial_candidates_required() -> None:
|
||||
scenario = _scenario()
|
||||
party = _party()
|
||||
with pytest.raises(ValueError, match="party_candidates must be non-empty"):
|
||||
adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=[],
|
||||
adversary_candidates=[AdversarySpec()],
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="adversary_candidates must be non-empty"):
|
||||
adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=[party],
|
||||
adversary_candidates=[],
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
|
||||
def test_adversarial_budget_is_enforced_before_any_battle() -> None:
|
||||
scenario = _scenario()
|
||||
party = _party()
|
||||
specs = (
|
||||
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
|
||||
AdversarySpec(name="plain"),
|
||||
)
|
||||
cost = 2 * (1 + len(specs)) * RUNS
|
||||
with pytest.raises(ValueError, match="robustness budget exceeded"):
|
||||
adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=[party],
|
||||
adversary_candidates=specs,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
iterations=2,
|
||||
max_total_runs=cost - 1,
|
||||
)
|
||||
entries = adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=[party],
|
||||
adversary_candidates=specs,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
iterations=2,
|
||||
max_total_runs=cost,
|
||||
)
|
||||
assert len(entries) == 2
|
||||
|
||||
|
||||
def test_danger_map_is_3x3_party_major_with_variant_names() -> None:
|
||||
scenario = _scenario()
|
||||
cells = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||
assert len(cells) == 9
|
||||
assert all(isinstance(cell, DangerCell) for cell in cells)
|
||||
order = [(cell.party_hp_fraction, cell.adversary_hp_fraction) for cell in cells]
|
||||
assert order == [(p, a) for p in (0.5, 0.7, 1.0) for a in (0.5, 0.7, 1.0)]
|
||||
assert cells[0].benchmark.scenario == "goblins-vs-orcs party_hp=0.5 adv_hp=0.5"
|
||||
assert cells[-1].benchmark.scenario == "goblins-vs-orcs party_hp=1.0 adv_hp=1.0"
|
||||
for cell in cells:
|
||||
assert 0.0 <= cell.benchmark.party_danger.p_survives <= 1.0
|
||||
|
||||
|
||||
def test_danger_map_rejects_out_of_range_fractions() -> None:
|
||||
scenario = _scenario()
|
||||
with pytest.raises(ValueError, match="fraction must be in"):
|
||||
danger_map(
|
||||
scenario, _party(), AdversarySpec(), fractions=(0.0,), runs=RUNS, seed=SEED
|
||||
)
|
||||
with pytest.raises(ValueError, match="fraction must be in"):
|
||||
danger_map(
|
||||
scenario, _party(), AdversarySpec(), fractions=(1.5,), runs=RUNS, seed=SEED
|
||||
)
|
||||
with pytest.raises(ValueError, match="fractions must be non-empty"):
|
||||
danger_map(
|
||||
scenario, _party(), AdversarySpec(), fractions=(), runs=RUNS, seed=SEED
|
||||
)
|
||||
|
||||
|
||||
def test_danger_map_is_reproducible() -> None:
|
||||
scenario = _scenario()
|
||||
first = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||
second = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_frontier_finds_reached_threshold() -> None:
|
||||
scenario = _scenario()
|
||||
result = frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
min_fraction=0.25,
|
||||
max_fraction=1.0,
|
||||
precision=0.05,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
# expected hp lost is always >= 0, so the threshold is crossed everywhere:
|
||||
# bisection converges to the smallest probed fraction, rounded to precision
|
||||
assert result == pytest.approx(0.25)
|
||||
|
||||
|
||||
def test_frontier_returns_none_when_threshold_unreachable() -> None:
|
||||
scenario = _scenario()
|
||||
result = frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=1e9,
|
||||
min_fraction=0.25,
|
||||
max_fraction=1.0,
|
||||
precision=0.05,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_frontier_is_reproducible() -> None:
|
||||
scenario = _scenario()
|
||||
first = frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
second = frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_frontier_returns_precision_multiple() -> None:
|
||||
scenario = _scenario()
|
||||
result = frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
min_fraction=0.3,
|
||||
max_fraction=1.0,
|
||||
precision=0.1,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
assert result is not None
|
||||
# bisection stops within precision of min_fraction; the result is the
|
||||
# nearest precision multiple in [min_fraction, min_fraction + precision]
|
||||
assert 0.3 <= result <= 0.4
|
||||
assert result == pytest.approx(round(result / 0.1) * 0.1)
|
||||
|
||||
|
||||
def test_frontier_p_survives_uses_inverse_threshold() -> None:
|
||||
scenario = _scenario()
|
||||
result = frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="p_survives",
|
||||
target=0.9999,
|
||||
min_fraction=0.5,
|
||||
max_fraction=1.0,
|
||||
precision=0.25,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
# convention: p_survives crosses when value <= target; a present result
|
||||
# is a precision multiple within the probed range
|
||||
if result is not None:
|
||||
assert 0.5 <= result <= 0.75
|
||||
assert result == pytest.approx(round(result / 0.25) * 0.25)
|
||||
|
||||
|
||||
def test_frontier_validates_arguments() -> None:
|
||||
scenario = _scenario()
|
||||
with pytest.raises(ValueError, match="criterion must be one of"):
|
||||
frontier(
|
||||
scenario, _party(), AdversarySpec(), criterion="bogus", runs=RUNS, seed=SEED
|
||||
)
|
||||
with pytest.raises(ValueError, match="target must be in"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="p_tpk",
|
||||
target=0.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="target must be in"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="p_survives",
|
||||
target=1.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="target must be >= 0"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=-1.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="min_fraction must be > 0"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
min_fraction=0.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="max_fraction must be <= 1"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
max_fraction=1.5,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="min_fraction must be <= max_fraction"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
min_fraction=0.9,
|
||||
max_fraction=0.5,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
with pytest.raises(ValueError, match="precision must be > 0"):
|
||||
frontier(
|
||||
scenario,
|
||||
_party(),
|
||||
AdversarySpec(),
|
||||
criterion="expected_hp_lost",
|
||||
target=0.0,
|
||||
precision=0.0,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
|
||||
def test_sensitivity_ranks_levers_and_reports_signed_deltas() -> None:
|
||||
scenario = _scenario()
|
||||
party = _party()
|
||||
entries = sensitivity(scenario, party, runs=RUNS, seed=SEED)
|
||||
assert len(entries) == 7
|
||||
assert all(isinstance(entry, SensitivityEntry) for entry in entries)
|
||||
assert {entry.levers for entry in entries} == {
|
||||
"weakest_ac",
|
||||
"lowest_hp",
|
||||
"healer",
|
||||
"spell_policy=none",
|
||||
"retreat=0.25",
|
||||
"retreat=0.5",
|
||||
"use_charges=false",
|
||||
}
|
||||
for previous, current in itertools.pairwise(entries):
|
||||
assert abs(previous.delta) >= abs(current.delta)
|
||||
if abs(previous.delta) == abs(current.delta):
|
||||
assert previous.levers <= current.levers
|
||||
# delta is the signed change vs the all-default baseline
|
||||
baseline = matchup_benchmark(
|
||||
scenario, party, AdversarySpec(), runs=RUNS, seed=SEED
|
||||
)
|
||||
base = baseline.party_danger.expected_hp_lost
|
||||
for entry in entries:
|
||||
assert entry.delta == pytest.approx(
|
||||
entry.benchmark.party_danger.expected_hp_lost - base
|
||||
)
|
||||
|
||||
|
||||
def test_sensitivity_rejects_unknown_criterion() -> None:
|
||||
scenario = _scenario()
|
||||
with pytest.raises(ValueError, match="criterion must be one of"):
|
||||
sensitivity(scenario, _party(), criterion="bogus", runs=RUNS, seed=SEED)
|
||||
|
||||
|
||||
def test_sensitivity_budget_is_enforced() -> None:
|
||||
scenario = _scenario()
|
||||
party = _party()
|
||||
with pytest.raises(ValueError, match="robustness budget exceeded"):
|
||||
sensitivity(
|
||||
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS - 1
|
||||
)
|
||||
entries = sensitivity(
|
||||
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS
|
||||
)
|
||||
assert len(entries) == 7
|
||||
|
||||
|
||||
def test_format_matchup_ascii_aligned() -> None:
|
||||
benchmark_ = matchup_benchmark(
|
||||
_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
|
||||
)
|
||||
text = format_matchup(benchmark_)
|
||||
lines = text.splitlines()
|
||||
assert lines[0].split() == [
|
||||
"Scenario",
|
||||
"Runs",
|
||||
"Side",
|
||||
"Win%",
|
||||
"Survive",
|
||||
"TPK",
|
||||
">=1-death",
|
||||
"HP-lost",
|
||||
"Rounds",
|
||||
]
|
||||
assert lines[1] == "-" * len(lines[0])
|
||||
assert "party" in lines[2]
|
||||
assert "adversary" in lines[3]
|
||||
assert all(len(line) == len(lines[0]) for line in lines)
|
||||
assert text.isascii()
|
||||
|
||||
|
||||
def test_format_robustness_ascii_aligned() -> None:
|
||||
scenario = _scenario()
|
||||
entries = adversarial_analysis(
|
||||
scenario,
|
||||
party_candidates=[_party()],
|
||||
adversary_candidates=[AdversarySpec()],
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
iterations=2,
|
||||
)
|
||||
text = format_robustness(entries)
|
||||
lines = text.splitlines()
|
||||
assert lines[0].split() == [
|
||||
"Iter",
|
||||
"Party",
|
||||
"Adversary",
|
||||
"P(party-survive)",
|
||||
"P(party-TPK)",
|
||||
"Party-HP",
|
||||
"P(adv-survive)",
|
||||
"P(adv-TPK)",
|
||||
"Adv-HP",
|
||||
]
|
||||
assert len(lines) == 2 + 2
|
||||
assert all(len(line) == len(lines[0]) for line in lines)
|
||||
assert text.isascii()
|
||||
|
||||
|
||||
def test_format_danger_map_ascii_aligned() -> None:
|
||||
cells = danger_map(_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||
text = format_danger_map(cells)
|
||||
lines = text.splitlines()
|
||||
assert lines[0].startswith("Party\\Adv")
|
||||
assert "adv=0.5" in lines[0]
|
||||
assert "party=0.5" in lines[2]
|
||||
assert "party=1.0" in lines[4]
|
||||
assert "/" in lines[2]
|
||||
assert all(len(line) == len(lines[0]) for line in lines)
|
||||
assert text.isascii()
|
||||
|
||||
|
||||
def test_format_frontier_messages() -> None:
|
||||
assert (
|
||||
format_frontier("p_tpk", 0.2, None, min_fraction=0.25, max_fraction=1.0)
|
||||
== "no frontier for p_tpk 0.2 in [0.25, 1.00]"
|
||||
)
|
||||
assert (
|
||||
format_frontier("p_tpk", 0.2, 0.35, min_fraction=0.25, max_fraction=1.0)
|
||||
== "frontier p_tpk 0.2: party hp_fraction = 0.35"
|
||||
)
|
||||
|
||||
|
||||
def test_format_sensitivity_ascii_aligned() -> None:
|
||||
entries = sensitivity(_scenario(), _party(), runs=RUNS, seed=SEED)
|
||||
text = format_sensitivity(entries, "expected_hp_lost")
|
||||
lines = text.splitlines()
|
||||
assert lines[0].split() == ["Rank", "Levers", "Value", "Delta"]
|
||||
assert len(lines) == 2 + 7
|
||||
assert lines[1].split()[-1].startswith(("+", "-"))
|
||||
assert all(len(line) == len(lines[0]) for line in lines)
|
||||
assert text.isascii()
|
||||
|
||||
|
||||
def test_robustness_module_does_not_import_search() -> None:
|
||||
source = Path(robustness.__file__).resolve().read_text(encoding="utf-8")
|
||||
assert "search" not in source
|
||||
Reference in New Issue
Block a user