feat(phase2): adversary and robustness cartography (P4)
This commit is contained in:
@@ -58,6 +58,9 @@ allowed-confusables = ["σ"]
|
|||||||
"src/pf1e_simulator/runner.py" = ["PLR0913"]
|
"src/pf1e_simulator/runner.py" = ["PLR0913"]
|
||||||
# client.py: LLMClient/FakeLLMClient need the full safeguard knobs (cache, budget, pricing, retry, breaker)
|
# client.py: LLMClient/FakeLLMClient need the full safeguard knobs (cache, budget, pricing, retry, breaker)
|
||||||
"src/pf1e_simulator/client.py" = ["PLR0913"]
|
"src/pf1e_simulator/client.py" = ["PLR0913"]
|
||||||
|
# robustness.py: the cartography API takes the full knob set (scenario, specs,
|
||||||
|
# side, runs, seed, fractions, target, precision, budget) as keyword parameters
|
||||||
|
"src/pf1e_simulator/robustness.py" = ["PLR0913"]
|
||||||
|
|
||||||
# ── Basedpyright ──────────────────────────────────────────────────────────────
|
# ── Basedpyright ──────────────────────────────────────────────────────────────
|
||||||
[tool.basedpyright]
|
[tool.basedpyright]
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""Adversary spec and compilation (docs/phase-2-llm-architecture.md section 10, P4.1).
|
||||||
|
|
||||||
|
The adversary-facing mirror of strategy.py: ``AdversarySpec`` is a
|
||||||
|
field-for-field copy of ``StrategySpec`` (same knobs, same validation) so the
|
||||||
|
enemy side of a matchup carries its own spec type without touching the party
|
||||||
|
search space. ``compile_adversary`` has identity semantics — it rebuilds the
|
||||||
|
mirrored ``StrategySpec`` and delegates to ``compile_strategy``, so the
|
||||||
|
all-default spec compiles to the module-level ``default_policy`` object
|
||||||
|
itself. ``compile_matchup`` glues the two sides into ONE ``Policy`` that
|
||||||
|
routes per combatant side: the engine invokes a single policy for every
|
||||||
|
turn (``CombatEngine._take_turn`` → ``_policy(self, state)``), so the policy
|
||||||
|
branches on ``state.side``, following the ``phase2_spike.focus_fire_policy``
|
||||||
|
pattern. Deterministic throughout: no RNG, no LLM.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from pf1e_simulator.combat import (
|
||||||
|
Action,
|
||||||
|
CombatantState,
|
||||||
|
CombatEngine,
|
||||||
|
Policy,
|
||||||
|
default_policy,
|
||||||
|
)
|
||||||
|
from pf1e_simulator.strategy import (
|
||||||
|
SpellPolicy,
|
||||||
|
StrategySpec,
|
||||||
|
TargetPriority,
|
||||||
|
compile_strategy,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AdversarySpec:
|
||||||
|
"""Field-for-field mirror of ``StrategySpec`` (same defaults, same validation)."""
|
||||||
|
|
||||||
|
name: str = "default"
|
||||||
|
target_priority: TargetPriority = "nearest"
|
||||||
|
retreat_hp_fraction: float = 0.0 # 0.0 = never retreat
|
||||||
|
spell_policy: SpellPolicy = "default"
|
||||||
|
use_charges: bool = True
|
||||||
|
use_class_abilities: bool = True
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if not self.name:
|
||||||
|
msg = "name must be a non-empty string"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if not 0.0 <= self.retreat_hp_fraction <= 1.0:
|
||||||
|
msg = "retreat_hp_fraction must be within [0, 1]"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def adversary_name(spec: AdversarySpec) -> str:
|
||||||
|
"""A deterministic label for `spec` from its non-default levers.
|
||||||
|
|
||||||
|
Local mirror of ``search.strategy_name`` (same field order: target_priority,
|
||||||
|
spell_policy, use_charges, use_class_abilities, retreat_hp_fraction; joined
|
||||||
|
with ", "; the `name` field never appears; all-default is "default"), kept
|
||||||
|
local so this module never imports search.py.
|
||||||
|
"""
|
||||||
|
levers: list[str] = []
|
||||||
|
if spec.target_priority != "nearest":
|
||||||
|
levers.append(spec.target_priority)
|
||||||
|
if spec.spell_policy != "default":
|
||||||
|
levers.append(f"spell_policy={spec.spell_policy}")
|
||||||
|
if not spec.use_charges:
|
||||||
|
levers.append("use_charges=false")
|
||||||
|
if not spec.use_class_abilities:
|
||||||
|
levers.append("use_class_abilities=false")
|
||||||
|
if spec.retreat_hp_fraction != 0.0:
|
||||||
|
levers.append(f"retreat={spec.retreat_hp_fraction}")
|
||||||
|
if not levers:
|
||||||
|
return "default"
|
||||||
|
return ", ".join(levers)
|
||||||
|
|
||||||
|
|
||||||
|
def compile_adversary(spec: AdversarySpec) -> Policy:
|
||||||
|
"""Compile an ``AdversarySpec`` into a ``Policy`` (identity semantics).
|
||||||
|
|
||||||
|
Rebuilds the mirrored ``StrategySpec`` and delegates to
|
||||||
|
``compile_strategy``, so ``compile_adversary(AdversarySpec()) is
|
||||||
|
default_policy`` holds exactly like the strategy-side identity.
|
||||||
|
"""
|
||||||
|
return compile_strategy(
|
||||||
|
StrategySpec(
|
||||||
|
name=spec.name,
|
||||||
|
target_priority=spec.target_priority,
|
||||||
|
retreat_hp_fraction=spec.retreat_hp_fraction,
|
||||||
|
spell_policy=spec.spell_policy,
|
||||||
|
use_charges=spec.use_charges,
|
||||||
|
use_class_abilities=spec.use_class_abilities,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def compile_matchup(
|
||||||
|
party: StrategySpec, adversary: AdversarySpec, *, party_side: str = "players"
|
||||||
|
) -> Policy:
|
||||||
|
"""One ``Policy`` that routes every turn by ``state.side`` (P4.1).
|
||||||
|
|
||||||
|
Turns of ``party_side`` delegate to ``compile_strategy(party)``; every
|
||||||
|
other side delegates to ``compile_adversary(adversary)`` — the engine
|
||||||
|
invokes ONE policy for all combatants, so side routing lives inside the
|
||||||
|
policy (the ``phase2_spike.focus_fire_policy`` pattern, generalised to
|
||||||
|
per-side compiled strategies). When both specs compile to
|
||||||
|
``default_policy``, the default object itself is returned (identity, no
|
||||||
|
wrapper). An empty ``party_side`` is a ValueError.
|
||||||
|
"""
|
||||||
|
if not party_side:
|
||||||
|
msg = "party_side must be a non-empty string"
|
||||||
|
raise ValueError(msg)
|
||||||
|
party_policy = compile_strategy(party)
|
||||||
|
adversary_policy = compile_adversary(adversary)
|
||||||
|
if party_policy is default_policy and adversary_policy is default_policy:
|
||||||
|
return default_policy
|
||||||
|
|
||||||
|
def policy(engine: CombatEngine, state: CombatantState) -> tuple[Action, ...]:
|
||||||
|
if state.side == party_side:
|
||||||
|
return party_policy(engine, state)
|
||||||
|
return adversary_policy(engine, state)
|
||||||
|
|
||||||
|
return policy
|
||||||
@@ -0,0 +1,682 @@
|
|||||||
|
"""Robustness layer: alternate min-max and danger cartography
|
||||||
|
(docs/phase-2-llm-architecture.md sections 3.2 and 10; P4.2 and P4.3).
|
||||||
|
|
||||||
|
A party's danger is only meaningful against an adversary that plays to
|
||||||
|
win (D12). This module implements the two robustness layers on top of
|
||||||
|
the compiled Etage-A pipeline:
|
||||||
|
|
||||||
|
- P4.2 - alternate min-max (section 3.2: max_PJ . min_monstres):
|
||||||
|
``adversarial_analysis`` alternately fixes one side and best-responds
|
||||||
|
the other (the party maximizes its own ``DangerMetrics``, then the
|
||||||
|
adversary minimizes the party's), every evaluation being a
|
||||||
|
``matchup_benchmark`` of one party ``StrategySpec`` vs one adversary
|
||||||
|
``AdversarySpec`` over a ``ScenarioSpec``. The same measure gives both
|
||||||
|
senses: danger for the party AND danger for the monsters.
|
||||||
|
- P4.3 - danger cartography (non exhaustive, D14): ``danger_map`` sweeps
|
||||||
|
a grid of per-side initial-HP fractions (``SideSpec.hp_fraction``) and
|
||||||
|
reports both sides' danger per cell, ``frontier`` bisects the party HP
|
||||||
|
fraction where a danger criterion crosses a threshold, and
|
||||||
|
``sensitivity`` ranks which single adversary lever moves the party
|
||||||
|
danger the most.
|
||||||
|
|
||||||
|
Deterministic throughout: no RNG, no LLM, no network. The engine's
|
||||||
|
``SeededRng`` under ``run_encounter`` is the only randomness and it is
|
||||||
|
seeded, so identical inputs produce bit-identical frozen results.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from pf1e_simulator.objectives import DangerMetrics
|
||||||
|
from pf1e_simulator.runner import EncounterReport
|
||||||
|
from pf1e_simulator.strategy import StrategySpec
|
||||||
|
|
||||||
|
from pf1e_simulator.adversary import AdversarySpec, adversary_name, compile_matchup
|
||||||
|
from pf1e_simulator.objectives import danger_metrics
|
||||||
|
from pf1e_simulator.runner import run_encounter
|
||||||
|
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec, build_encounter
|
||||||
|
|
||||||
|
_CRITERIA = ("p_survives", "p_tpk", "p_any_death", "expected_hp_lost")
|
||||||
|
_MATCHUP_SIDES = 2 # a matchup is exactly one arena pair (two sides)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_criterion(criterion: str) -> None:
|
||||||
|
if criterion not in _CRITERIA:
|
||||||
|
msg = (
|
||||||
|
"criterion must be one of ('p_survives', 'p_tpk', 'p_any_death', "
|
||||||
|
f"'expected_hp_lost'), got {criterion!r}"
|
||||||
|
)
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _danger_value(danger: DangerMetrics, criterion: str) -> float:
|
||||||
|
"""The `criterion` value of a danger profile (explicit dispatch, no getattr)."""
|
||||||
|
if criterion == "p_survives":
|
||||||
|
return danger.p_survives
|
||||||
|
if criterion == "p_tpk":
|
||||||
|
return danger.p_tpk
|
||||||
|
if criterion == "p_any_death":
|
||||||
|
return danger.p_any_death
|
||||||
|
return danger.expected_hp_lost
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class MatchupBenchmark:
|
||||||
|
"""One (party strategy x adversary spec) matchup over a scenario.
|
||||||
|
|
||||||
|
``report`` aggregates all runs; ``party_danger`` is the danger of the
|
||||||
|
``party_side`` side and ``adversary_danger`` the danger of the other
|
||||||
|
side (same measure, both senses: PJ and monsters).
|
||||||
|
"""
|
||||||
|
|
||||||
|
scenario: str
|
||||||
|
party: str
|
||||||
|
adversary: str
|
||||||
|
runs: int
|
||||||
|
seed: int
|
||||||
|
report: EncounterReport
|
||||||
|
party_danger: DangerMetrics
|
||||||
|
adversary_danger: DangerMetrics
|
||||||
|
party_side: str
|
||||||
|
|
||||||
|
|
||||||
|
def _other_side(scenario: ScenarioSpec, party_side: str) -> str:
|
||||||
|
"""The unique side different from ``party_side`` (exactly-2-sides contract)."""
|
||||||
|
names = tuple(side.name for side in scenario.sides)
|
||||||
|
if len(names) != _MATCHUP_SIDES:
|
||||||
|
msg = f"matchup requires exactly 2 sides, got {len(names)}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if party_side not in names:
|
||||||
|
msg = f"party_side {party_side!r} is not a side of the scenario"
|
||||||
|
raise ValueError(msg)
|
||||||
|
(other,) = (name for name in names if name != party_side)
|
||||||
|
return other
|
||||||
|
|
||||||
|
|
||||||
|
def matchup_benchmark(
|
||||||
|
scenario: ScenarioSpec,
|
||||||
|
party: StrategySpec,
|
||||||
|
adversary: AdversarySpec,
|
||||||
|
*,
|
||||||
|
party_side: str = "players",
|
||||||
|
runs: int = 100,
|
||||||
|
seed: int = 1,
|
||||||
|
) -> MatchupBenchmark:
|
||||||
|
"""Compile one matchup and measure both sides' danger over `runs` battles.
|
||||||
|
|
||||||
|
Deterministic: same inputs -> bit-identical ``MatchupBenchmark``
|
||||||
|
(frozen). Exactly two sides are required (one arena pair), the party
|
||||||
|
side name must be non-empty and be one of them, and ``runs`` >= 1.
|
||||||
|
"""
|
||||||
|
if not party_side:
|
||||||
|
msg = "party_side must be a non-empty string"
|
||||||
|
raise ValueError(msg)
|
||||||
|
adversary_side = _other_side(scenario, party_side)
|
||||||
|
if runs < 1:
|
||||||
|
msg = "runs must be >= 1"
|
||||||
|
raise ValueError(msg)
|
||||||
|
encounter = build_encounter(scenario)
|
||||||
|
policy = compile_matchup(party, adversary, party_side=party_side)
|
||||||
|
report, outcomes = run_encounter(
|
||||||
|
encounter, runs, seed, policy=policy, collect_runs=True
|
||||||
|
)
|
||||||
|
return MatchupBenchmark(
|
||||||
|
scenario=scenario.name,
|
||||||
|
party=party.name,
|
||||||
|
adversary=adversary.name,
|
||||||
|
runs=runs,
|
||||||
|
seed=seed,
|
||||||
|
report=report,
|
||||||
|
party_danger=danger_metrics(outcomes, party_side),
|
||||||
|
adversary_danger=danger_metrics(outcomes, adversary_side),
|
||||||
|
party_side=party_side,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RobustnessEntry:
|
||||||
|
"""One turn of the alternating min-max: the pair kept after that turn."""
|
||||||
|
|
||||||
|
iteration: int
|
||||||
|
party: StrategySpec
|
||||||
|
adversary: AdversarySpec
|
||||||
|
benchmark: MatchupBenchmark
|
||||||
|
|
||||||
|
|
||||||
|
def _party_rank(benchmark: MatchupBenchmark) -> tuple[float, float, str]:
|
||||||
|
"""Key: the party best-response prefers high survival, low HP lost, then name."""
|
||||||
|
danger = benchmark.party_danger
|
||||||
|
return (danger.p_survives, -danger.expected_hp_lost, benchmark.party)
|
||||||
|
|
||||||
|
|
||||||
|
def _adversary_rank(benchmark: MatchupBenchmark) -> tuple[float, float, str]:
|
||||||
|
"""Key: the adversary best-response prefers low party survival, high HP lost."""
|
||||||
|
danger = benchmark.party_danger
|
||||||
|
return (danger.p_survives, -danger.expected_hp_lost, benchmark.adversary)
|
||||||
|
|
||||||
|
|
||||||
|
def adversarial_analysis(
|
||||||
|
scenario: ScenarioSpec,
|
||||||
|
*,
|
||||||
|
party_candidates: Sequence[StrategySpec],
|
||||||
|
adversary_candidates: Sequence[AdversarySpec],
|
||||||
|
party_side: str = "players",
|
||||||
|
runs: int = 100,
|
||||||
|
seed: int = 1,
|
||||||
|
iterations: int = 3,
|
||||||
|
max_total_runs: int | None = None,
|
||||||
|
) -> tuple[RobustnessEntry, ...]:
|
||||||
|
"""Alternate (party, adversary) best responses over `iterations` turns.
|
||||||
|
|
||||||
|
Each turn re-evaluates every party candidate against the current
|
||||||
|
adversary and keeps the best for the party (max own survival), then
|
||||||
|
every adversary candidate against the kept party and keeps the worst
|
||||||
|
for the party (min own survival, max party HP lost) -- the min-max
|
||||||
|
alternation of section 3.2. The kept pair is reported with its
|
||||||
|
benchmark.
|
||||||
|
|
||||||
|
Budget: at most ``iterations * (party + adversary candidates) * runs``
|
||||||
|
battles. When ``max_total_runs`` is set and the budget exceeds it, no
|
||||||
|
battle is fought and a ValueError is raised.
|
||||||
|
"""
|
||||||
|
if not party_candidates:
|
||||||
|
msg = "party_candidates must be non-empty"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if not adversary_candidates:
|
||||||
|
msg = "adversary_candidates must be non-empty"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if iterations < 1:
|
||||||
|
msg = f"iterations must be >= 1, got {iterations}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if runs < 1:
|
||||||
|
msg = f"runs must be >= 1, got {runs}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
cost = iterations * (len(party_candidates) + len(adversary_candidates)) * runs
|
||||||
|
if max_total_runs is not None and cost > max_total_runs:
|
||||||
|
msg = (
|
||||||
|
f"robustness budget exceeded: {cost} runs > "
|
||||||
|
f"{max_total_runs} max_total_runs"
|
||||||
|
)
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
party = party_candidates[0]
|
||||||
|
adversary = adversary_candidates[0]
|
||||||
|
entries: list[RobustnessEntry] = []
|
||||||
|
for iteration in range(1, iterations + 1):
|
||||||
|
party_pairs: list[tuple[StrategySpec, MatchupBenchmark]] = [
|
||||||
|
(
|
||||||
|
candidate,
|
||||||
|
matchup_benchmark(
|
||||||
|
scenario,
|
||||||
|
candidate,
|
||||||
|
adversary,
|
||||||
|
party_side=party_side,
|
||||||
|
runs=runs,
|
||||||
|
seed=seed,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for candidate in party_candidates
|
||||||
|
]
|
||||||
|
party = max(party_pairs, key=lambda pair: _party_rank(pair[1]))[0]
|
||||||
|
adversary_pairs: list[tuple[AdversarySpec, MatchupBenchmark]] = [
|
||||||
|
(
|
||||||
|
candidate,
|
||||||
|
matchup_benchmark(
|
||||||
|
scenario,
|
||||||
|
party,
|
||||||
|
candidate,
|
||||||
|
party_side=party_side,
|
||||||
|
runs=runs,
|
||||||
|
seed=seed,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for candidate in adversary_candidates
|
||||||
|
]
|
||||||
|
adversary, benchmark = min(
|
||||||
|
adversary_pairs, key=lambda pair: _adversary_rank(pair[1])
|
||||||
|
)
|
||||||
|
entries.append(
|
||||||
|
RobustnessEntry(
|
||||||
|
iteration=iteration,
|
||||||
|
party=party,
|
||||||
|
adversary=adversary,
|
||||||
|
benchmark=benchmark,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(entries)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DangerCell:
|
||||||
|
"""One (party HP x adversary HP) cell of the danger map."""
|
||||||
|
|
||||||
|
party_hp_fraction: float
|
||||||
|
adversary_hp_fraction: float
|
||||||
|
benchmark: MatchupBenchmark
|
||||||
|
|
||||||
|
|
||||||
|
def _side_variant(
|
||||||
|
scenario: ScenarioSpec,
|
||||||
|
*,
|
||||||
|
party_side: str,
|
||||||
|
party_hp_fraction: float,
|
||||||
|
adversary_hp_fraction: float,
|
||||||
|
) -> ScenarioSpec:
|
||||||
|
"""A deterministic clone of `scenario` with per-side initial-HP fractions.
|
||||||
|
|
||||||
|
The `party_side` side carries `party_hp_fraction`; every other side
|
||||||
|
carries `adversary_hp_fraction` (the matchup contract is exactly 2
|
||||||
|
sides). Name: "<base> party_hp=<f> adv_hp=<g>" so cells stay distinct.
|
||||||
|
"""
|
||||||
|
sides = tuple(
|
||||||
|
SideSpec(
|
||||||
|
name=side.name,
|
||||||
|
zone=side.zone,
|
||||||
|
combatant_files=side.combatant_files,
|
||||||
|
hp_fraction=(
|
||||||
|
party_hp_fraction if side.name == party_side else adversary_hp_fraction
|
||||||
|
),
|
||||||
|
aware=side.aware,
|
||||||
|
)
|
||||||
|
for side in scenario.sides
|
||||||
|
)
|
||||||
|
return ScenarioSpec(
|
||||||
|
name=(
|
||||||
|
f"{scenario.name} party_hp={party_hp_fraction} "
|
||||||
|
f"adv_hp={adversary_hp_fraction}"
|
||||||
|
),
|
||||||
|
map_file=scenario.map_file,
|
||||||
|
sides=sides,
|
||||||
|
round_cap=scenario.round_cap,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def danger_map(
|
||||||
|
scenario: ScenarioSpec,
|
||||||
|
party: StrategySpec,
|
||||||
|
adversary: AdversarySpec,
|
||||||
|
*,
|
||||||
|
party_side: str = "players",
|
||||||
|
fractions: Sequence[float] = (0.5, 0.7, 1.0),
|
||||||
|
runs: int = 100,
|
||||||
|
seed: int = 1,
|
||||||
|
) -> tuple[DangerCell, ...]:
|
||||||
|
"""The grid of party HP x adversary HP danger cells (non exhaustive, D14).
|
||||||
|
|
||||||
|
One ``DangerCell`` per combination, party-major (party fraction
|
||||||
|
outermost), each with a distinct scenario name
|
||||||
|
"<base> party_hp=<f> adv_hp=<g>". Every fraction must lie in (0, 1].
|
||||||
|
"""
|
||||||
|
if not fractions:
|
||||||
|
msg = "fractions must be non-empty"
|
||||||
|
raise ValueError(msg)
|
||||||
|
for fraction in fractions:
|
||||||
|
if not 0.0 < fraction <= 1.0:
|
||||||
|
msg = f"fraction must be in (0, 1], got {fraction}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
return tuple(
|
||||||
|
DangerCell(
|
||||||
|
party_hp_fraction=party_frac,
|
||||||
|
adversary_hp_fraction=adv_frac,
|
||||||
|
benchmark=matchup_benchmark(
|
||||||
|
_side_variant(
|
||||||
|
scenario,
|
||||||
|
party_side=party_side,
|
||||||
|
party_hp_fraction=party_frac,
|
||||||
|
adversary_hp_fraction=adv_frac,
|
||||||
|
),
|
||||||
|
party,
|
||||||
|
adversary,
|
||||||
|
party_side=party_side,
|
||||||
|
runs=runs,
|
||||||
|
seed=seed,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for party_frac in fractions
|
||||||
|
for adv_frac in fractions
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_frontier(
|
||||||
|
criterion: str,
|
||||||
|
target: float,
|
||||||
|
min_fraction: float,
|
||||||
|
max_fraction: float,
|
||||||
|
precision: float,
|
||||||
|
) -> None:
|
||||||
|
"""Validate frontier parameters (keeps ``frontier`` itself flat)."""
|
||||||
|
_check_criterion(criterion)
|
||||||
|
if criterion == "expected_hp_lost":
|
||||||
|
if target < 0:
|
||||||
|
msg = f"target must be >= 0, got {target}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
elif not 0.0 < target < 1.0:
|
||||||
|
msg = f"target must be in (0, 1), got {target}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if min_fraction <= 0:
|
||||||
|
msg = f"min_fraction must be > 0, got {min_fraction}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if max_fraction > 1:
|
||||||
|
msg = f"max_fraction must be <= 1, got {max_fraction}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if min_fraction > max_fraction:
|
||||||
|
msg = (
|
||||||
|
f"min_fraction must be <= max_fraction, "
|
||||||
|
f"got min={min_fraction} max={max_fraction}"
|
||||||
|
)
|
||||||
|
raise ValueError(msg)
|
||||||
|
if precision <= 0:
|
||||||
|
msg = f"precision must be > 0, got {precision}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def frontier(
|
||||||
|
scenario: ScenarioSpec,
|
||||||
|
party: StrategySpec,
|
||||||
|
adversary: AdversarySpec,
|
||||||
|
*,
|
||||||
|
party_side: str = "players",
|
||||||
|
criterion: str = "p_tpk",
|
||||||
|
target: float = 0.2,
|
||||||
|
min_fraction: float = 0.25,
|
||||||
|
max_fraction: float = 1.0,
|
||||||
|
precision: float = 0.05,
|
||||||
|
runs: int = 100,
|
||||||
|
seed: int = 1,
|
||||||
|
) -> float | None:
|
||||||
|
"""The smallest party HP fraction where `criterion` crosses `target`.
|
||||||
|
|
||||||
|
The adversary HP fraction stays at 1.0. Crossing convention (section
|
||||||
|
3.2): ``value >= target`` for p_tpk, p_any_death and expected_hp_lost;
|
||||||
|
``value <= target`` for p_survives. Monotone danger (less party HP is
|
||||||
|
strictly worse): a crossing at ``min_fraction`` already yields the
|
||||||
|
smallest probed fraction, ceiled to the ``precision`` grid; otherwise
|
||||||
|
bisection converges to the smallest crossing fraction and returns
|
||||||
|
``ceil(hi / precision) * precision`` (ceiling keeps the fraction on
|
||||||
|
the crossing side of the threshold). None when not even
|
||||||
|
``max_fraction`` crosses. Probabilities require ``0 < target < 1``;
|
||||||
|
expected_hp_lost requires ``target >= 0``.
|
||||||
|
"""
|
||||||
|
_validate_frontier(criterion, target, min_fraction, max_fraction, precision)
|
||||||
|
|
||||||
|
def _crossed(fraction: float) -> bool:
|
||||||
|
cell = matchup_benchmark(
|
||||||
|
_side_variant(
|
||||||
|
scenario,
|
||||||
|
party_side=party_side,
|
||||||
|
party_hp_fraction=fraction,
|
||||||
|
adversary_hp_fraction=1.0,
|
||||||
|
),
|
||||||
|
party,
|
||||||
|
adversary,
|
||||||
|
party_side=party_side,
|
||||||
|
runs=runs,
|
||||||
|
seed=seed,
|
||||||
|
)
|
||||||
|
value = _danger_value(cell.party_danger, criterion)
|
||||||
|
if criterion == "p_survives":
|
||||||
|
return value <= target
|
||||||
|
return value >= target
|
||||||
|
|
||||||
|
if _crossed(min_fraction):
|
||||||
|
return math.ceil(min_fraction / precision) * precision
|
||||||
|
if not _crossed(max_fraction):
|
||||||
|
return None
|
||||||
|
lo = min_fraction
|
||||||
|
hi = max_fraction
|
||||||
|
while hi - lo > precision:
|
||||||
|
mid = (lo + hi) / 2.0
|
||||||
|
if _crossed(mid):
|
||||||
|
hi = mid
|
||||||
|
else:
|
||||||
|
lo = mid
|
||||||
|
return math.ceil(hi / precision) * precision
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SensitivityEntry:
|
||||||
|
"""One adversary lever raised from the baseline, with its danger delta."""
|
||||||
|
|
||||||
|
levers: str
|
||||||
|
spec: AdversarySpec
|
||||||
|
benchmark: MatchupBenchmark
|
||||||
|
delta: float
|
||||||
|
|
||||||
|
|
||||||
|
def sensitivity(
|
||||||
|
scenario: ScenarioSpec,
|
||||||
|
party: StrategySpec,
|
||||||
|
*,
|
||||||
|
party_side: str = "players",
|
||||||
|
criterion: str = "expected_hp_lost",
|
||||||
|
runs: int = 100,
|
||||||
|
seed: int = 1,
|
||||||
|
max_total_runs: int | None = None,
|
||||||
|
) -> tuple[SensitivityEntry, ...]:
|
||||||
|
"""Rank which single adversary lever moves the party danger the most.
|
||||||
|
|
||||||
|
Baseline is the all-default ``AdversarySpec()``; each of the 7
|
||||||
|
candidate levers varies exactly one knob (target_priority x3,
|
||||||
|
spell_policy=none, retreat x2, use_charges=false). ``delta`` is the
|
||||||
|
signed change of the party ``criterion`` value vs the baseline
|
||||||
|
(positive = more dangerous). Sorted by |delta| descending, ties by
|
||||||
|
lever label ascending. Budget is exactly 8 x runs (baseline + 7
|
||||||
|
levers), enforced before any battle.
|
||||||
|
"""
|
||||||
|
_check_criterion(criterion)
|
||||||
|
cost = 8 * runs
|
||||||
|
if max_total_runs is not None and cost > max_total_runs:
|
||||||
|
msg = (
|
||||||
|
f"robustness budget exceeded: {cost} runs > "
|
||||||
|
f"{max_total_runs} max_total_runs"
|
||||||
|
)
|
||||||
|
raise ValueError(msg)
|
||||||
|
levers = (
|
||||||
|
AdversarySpec(target_priority="weakest_ac"),
|
||||||
|
AdversarySpec(target_priority="lowest_hp"),
|
||||||
|
AdversarySpec(target_priority="healer"),
|
||||||
|
AdversarySpec(spell_policy="none"),
|
||||||
|
AdversarySpec(retreat_hp_fraction=0.25),
|
||||||
|
AdversarySpec(retreat_hp_fraction=0.5),
|
||||||
|
AdversarySpec(use_charges=False),
|
||||||
|
)
|
||||||
|
baseline = matchup_benchmark(
|
||||||
|
scenario,
|
||||||
|
party,
|
||||||
|
AdversarySpec(),
|
||||||
|
party_side=party_side,
|
||||||
|
runs=runs,
|
||||||
|
seed=seed,
|
||||||
|
)
|
||||||
|
base = _danger_value(baseline.party_danger, criterion)
|
||||||
|
entries: list[SensitivityEntry] = []
|
||||||
|
for spec in levers:
|
||||||
|
benchmark = matchup_benchmark(
|
||||||
|
scenario, party, spec, party_side=party_side, runs=runs, seed=seed
|
||||||
|
)
|
||||||
|
entries.append(
|
||||||
|
SensitivityEntry(
|
||||||
|
levers=adversary_name(spec),
|
||||||
|
spec=spec,
|
||||||
|
benchmark=benchmark,
|
||||||
|
delta=_danger_value(benchmark.party_danger, criterion) - base,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
entries.sort(key=lambda entry: (-abs(entry.delta), entry.levers))
|
||||||
|
return tuple(entries)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_line(values: Sequence[str], widths: Sequence[int]) -> str:
|
||||||
|
"""One aligned row: every cell ljust-padded to its column width."""
|
||||||
|
return " ".join(value.ljust(width) for value, width in zip(values, widths, strict=True))
|
||||||
|
|
||||||
|
|
||||||
|
def _render_table(headers: Sequence[str], rows: Sequence[Sequence[str]]) -> str:
|
||||||
|
"""A plain-text grid: header, a full-width dash separator, data rows.
|
||||||
|
|
||||||
|
Column widths are the max width over header and rows (same algorithm
|
||||||
|
as ``sweep.format_sweep``), so every line has identical length and the
|
||||||
|
separator spans the whole header line.
|
||||||
|
"""
|
||||||
|
columns: tuple[Sequence[str], ...] = (headers, *rows)
|
||||||
|
widths = tuple(
|
||||||
|
max(len(row[column]) for row in columns) for column in range(len(headers))
|
||||||
|
)
|
||||||
|
header = _render_line(headers, widths)
|
||||||
|
lines = [header, "-" * len(header)]
|
||||||
|
lines.extend(_render_line(row, widths) for row in rows)
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def format_matchup(benchmark: MatchupBenchmark) -> str:
|
||||||
|
"""Render one matchup: per-side win rate, danger profile and rounds."""
|
||||||
|
headers = (
|
||||||
|
"Scenario",
|
||||||
|
"Runs",
|
||||||
|
"Side",
|
||||||
|
"Win%",
|
||||||
|
"Survive",
|
||||||
|
"TPK",
|
||||||
|
">=1-death",
|
||||||
|
"HP-lost",
|
||||||
|
"Rounds",
|
||||||
|
)
|
||||||
|
adversary_side = next(
|
||||||
|
side for side in benchmark.report.wins if side != benchmark.party_side
|
||||||
|
)
|
||||||
|
party_row = (
|
||||||
|
benchmark.scenario,
|
||||||
|
str(benchmark.runs),
|
||||||
|
"party",
|
||||||
|
f"{benchmark.report.win_rate(benchmark.party_side):.0%}",
|
||||||
|
f"{benchmark.party_danger.p_survives:.0%}",
|
||||||
|
f"{benchmark.party_danger.p_tpk:.0%}",
|
||||||
|
f"{benchmark.party_danger.p_any_death:.0%}",
|
||||||
|
f"{benchmark.party_danger.expected_hp_lost:.1f}",
|
||||||
|
f"{benchmark.report.avg_rounds:.1f}",
|
||||||
|
)
|
||||||
|
adversary_row = (
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"adversary",
|
||||||
|
f"{benchmark.report.win_rate(adversary_side):.0%}",
|
||||||
|
f"{benchmark.adversary_danger.p_survives:.0%}",
|
||||||
|
f"{benchmark.adversary_danger.p_tpk:.0%}",
|
||||||
|
f"{benchmark.adversary_danger.p_any_death:.0%}",
|
||||||
|
f"{benchmark.adversary_danger.expected_hp_lost:.1f}",
|
||||||
|
f"{benchmark.report.avg_rounds:.1f}",
|
||||||
|
)
|
||||||
|
return _render_table(headers, (party_row, adversary_row))
|
||||||
|
|
||||||
|
|
||||||
|
def format_robustness(entries: Sequence[RobustnessEntry]) -> str:
|
||||||
|
"""Render the alternating min-max log, one aligned line per turn."""
|
||||||
|
headers = (
|
||||||
|
"Iter",
|
||||||
|
"Party",
|
||||||
|
"Adversary",
|
||||||
|
"P(party-survive)",
|
||||||
|
"P(party-TPK)",
|
||||||
|
"Party-HP",
|
||||||
|
"P(adv-survive)",
|
||||||
|
"P(adv-TPK)",
|
||||||
|
"Adv-HP",
|
||||||
|
)
|
||||||
|
rows = tuple(
|
||||||
|
(
|
||||||
|
str(entry.iteration),
|
||||||
|
entry.party.name,
|
||||||
|
entry.adversary.name,
|
||||||
|
f"{entry.benchmark.party_danger.p_survives:.0%}",
|
||||||
|
f"{entry.benchmark.party_danger.p_tpk:.0%}",
|
||||||
|
f"{entry.benchmark.party_danger.expected_hp_lost:.1f}",
|
||||||
|
f"{entry.benchmark.adversary_danger.p_survives:.0%}",
|
||||||
|
f"{entry.benchmark.adversary_danger.p_tpk:.0%}",
|
||||||
|
f"{entry.benchmark.adversary_danger.expected_hp_lost:.1f}",
|
||||||
|
)
|
||||||
|
for entry in entries
|
||||||
|
)
|
||||||
|
return _render_table(headers, rows)
|
||||||
|
|
||||||
|
|
||||||
|
def format_danger_map(cells: Sequence[DangerCell]) -> str:
|
||||||
|
"""Render the danger grid: rows = party HP, columns = adversary HP.
|
||||||
|
|
||||||
|
Each cell shows party survive% / adversary survive%, both as 0%
|
||||||
|
percentages (order preserved from ``danger_map``, hence the
|
||||||
|
dict.fromkeys dedupe instead of a set).
|
||||||
|
"""
|
||||||
|
party_fractions: tuple[float, ...] = tuple(
|
||||||
|
dict.fromkeys(cell.party_hp_fraction for cell in cells)
|
||||||
|
)
|
||||||
|
adversary_fractions: tuple[float, ...] = tuple(
|
||||||
|
dict.fromkeys(cell.adversary_hp_fraction for cell in cells)
|
||||||
|
)
|
||||||
|
by_cell: dict[tuple[float, float], DangerCell] = {
|
||||||
|
(cell.party_hp_fraction, cell.adversary_hp_fraction): cell for cell in cells
|
||||||
|
}
|
||||||
|
headers = (
|
||||||
|
"Party\\Adv",
|
||||||
|
*(f"adv={fraction}" for fraction in adversary_fractions),
|
||||||
|
)
|
||||||
|
rows: list[tuple[str, ...]] = []
|
||||||
|
for fraction in party_fractions:
|
||||||
|
row: list[str] = [f"party={fraction}"]
|
||||||
|
for adv in adversary_fractions:
|
||||||
|
cell = by_cell[(fraction, adv)]
|
||||||
|
row.append(
|
||||||
|
f"{cell.benchmark.party_danger.p_survives:.0%}/"
|
||||||
|
f"{cell.benchmark.adversary_danger.p_survives:.0%}"
|
||||||
|
)
|
||||||
|
rows.append(tuple(row))
|
||||||
|
return _render_table(headers, rows)
|
||||||
|
|
||||||
|
|
||||||
|
def format_frontier(
|
||||||
|
criterion: str,
|
||||||
|
target: float,
|
||||||
|
fraction: float | None,
|
||||||
|
*,
|
||||||
|
min_fraction: float,
|
||||||
|
max_fraction: float,
|
||||||
|
) -> str:
|
||||||
|
"""Render a frontier result (None -> "no frontier" message)."""
|
||||||
|
if fraction is None:
|
||||||
|
return (
|
||||||
|
f"no frontier for {criterion} {target} "
|
||||||
|
f"in [{min_fraction:.2f}, {max_fraction:.2f}]"
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
f"frontier {criterion} {target}: "
|
||||||
|
f"party hp_fraction = {fraction:.2f}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _format_criterion_value(danger: DangerMetrics, criterion: str) -> str:
|
||||||
|
"""One danger criterion as text: 0% for probabilities, 1 decimal for HP."""
|
||||||
|
value = _danger_value(danger, criterion)
|
||||||
|
if criterion == "expected_hp_lost":
|
||||||
|
return f"{value:.1f}"
|
||||||
|
return f"{value:.0%}"
|
||||||
|
|
||||||
|
|
||||||
|
def format_sensitivity(entries: Sequence[SensitivityEntry], criterion: str) -> str:
|
||||||
|
"""Render the lever ranking: rank, levers, party criterion value, delta."""
|
||||||
|
headers = ("Rank", "Levers", "Value", "Delta")
|
||||||
|
probability = criterion != "expected_hp_lost"
|
||||||
|
rows = tuple(
|
||||||
|
(
|
||||||
|
str(index),
|
||||||
|
entry.levers,
|
||||||
|
_format_criterion_value(entry.benchmark.party_danger, criterion),
|
||||||
|
f"{entry.delta:+.0%}" if probability else f"{entry.delta:+.1f}",
|
||||||
|
)
|
||||||
|
for index, entry in enumerate(entries, start=1)
|
||||||
|
)
|
||||||
|
return _render_table(headers, rows)
|
||||||
@@ -35,6 +35,7 @@ class SideSpec:
|
|||||||
zone: str # deployment zone letter, e.g. "A"
|
zone: str # deployment zone letter, e.g. "A"
|
||||||
combatant_files: tuple[Path, ...] # monster/PJ JSON files, loaded via load_combatant
|
combatant_files: tuple[Path, ...] # monster/PJ JSON files, loaded via load_combatant
|
||||||
hp_fraction: float = 1.0 # initial HP as a fraction of hp_max, in (0, 1]
|
hp_fraction: float = 1.0 # initial HP as a fraction of hp_max, in (0, 1]
|
||||||
|
aware: bool | None = None # SRD "Surprise" override; None keeps Combatant.aware (True)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -82,6 +83,19 @@ def _apply_hp_fraction(combatant: Combatant, fraction: float) -> Combatant:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_aware(combatant: Combatant, *, aware: bool | None) -> Combatant:
|
||||||
|
"""Apply a side's SRD "Surprise" awareness override via model_copy.
|
||||||
|
|
||||||
|
None keeps the loaded combatant unchanged (Combatant.aware defaults to
|
||||||
|
True, models.py:416-420). SRD "Surprise": only some combatants aware
|
||||||
|
means a surprise round precedes round 1; unaware combatants do not act
|
||||||
|
in it.
|
||||||
|
"""
|
||||||
|
if aware is None:
|
||||||
|
return combatant
|
||||||
|
return combatant.model_copy(update={"aware": aware})
|
||||||
|
|
||||||
|
|
||||||
def build_encounter(scenario: ScenarioSpec) -> EncounterSpec:
|
def build_encounter(scenario: ScenarioSpec) -> EncounterSpec:
|
||||||
"""Compile a ScenarioSpec into a runner EncounterSpec (deterministic).
|
"""Compile a ScenarioSpec into a runner EncounterSpec (deterministic).
|
||||||
|
|
||||||
@@ -89,6 +103,12 @@ def build_encounter(scenario: ScenarioSpec) -> EncounterSpec:
|
|||||||
file with load_combatant, then apply hp_fraction to hp_max via
|
file with load_combatant, then apply hp_fraction to hp_max via
|
||||||
pydantic model_copy(update={"hp_max": max(1, round(hp_max * fraction))})
|
pydantic model_copy(update={"hp_max": max(1, round(hp_max * fraction))})
|
||||||
(same model_copy pattern as runner.build_states).
|
(same model_copy pattern as runner.build_states).
|
||||||
|
- When a side's aware is not None, it is applied to that side's
|
||||||
|
combatants via model_copy(update={"aware": ...}) at the same point
|
||||||
|
where hp_fraction is applied (SRD "Surprise" round semantics: only
|
||||||
|
some aware -> surprise round; Combatant.aware defaults to True, so
|
||||||
|
aware=None keeps the loaded default and is bit-identical to the
|
||||||
|
behaviour without the field).
|
||||||
- Duplicate combatant ids are NOT disambiguated here: build_states
|
- Duplicate combatant ids are NOT disambiguated here: build_states
|
||||||
(called later by run_encounter) handles that. Just list each loaded
|
(called later by run_encounter) handles that. Just list each loaded
|
||||||
combatant once per occurrence in combatant_files.
|
combatant once per occurrence in combatant_files.
|
||||||
@@ -126,7 +146,10 @@ def build_encounter(scenario: ScenarioSpec) -> EncounterSpec:
|
|||||||
Side(
|
Side(
|
||||||
name=side.name,
|
name=side.name,
|
||||||
combatants=tuple(
|
combatants=tuple(
|
||||||
_apply_hp_fraction(load_combatant(path), side.hp_fraction)
|
_apply_aware(
|
||||||
|
_apply_hp_fraction(load_combatant(path), side.hp_fraction),
|
||||||
|
aware=side.aware,
|
||||||
|
)
|
||||||
for path in side.combatant_files
|
for path in side.combatant_files
|
||||||
),
|
),
|
||||||
zone=side.zone,
|
zone=side.zone,
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ from pf1e_simulator.combat import (
|
|||||||
default_policy,
|
default_policy,
|
||||||
)
|
)
|
||||||
|
|
||||||
TargetPriority = Literal["nearest", "weakest_ac", "lowest_hp"]
|
TargetPriority = Literal["nearest", "weakest_ac", "lowest_hp", "healer"]
|
||||||
SpellPolicy = Literal["default", "none"]
|
SpellPolicy = Literal["default", "none"]
|
||||||
|
|
||||||
|
|
||||||
@@ -83,6 +83,16 @@ def select_target(
|
|||||||
)
|
)
|
||||||
case "lowest_hp":
|
case "lowest_hp":
|
||||||
return min(enemies, key=lambda s: (s.hp, engine.states.index(s)))
|
return min(enemies, key=lambda s: (s.hp, engine.states.index(s)))
|
||||||
|
case "healer":
|
||||||
|
# Prefer the caster/healer (non-empty combatant.spells); among
|
||||||
|
# several, min by (ac, hp, list order). No active caster on the
|
||||||
|
# enemy side -> identical to weakest_ac.
|
||||||
|
casters = [s for s in enemies if s.combatant.spells]
|
||||||
|
pool = casters or enemies
|
||||||
|
return min(
|
||||||
|
pool,
|
||||||
|
key=lambda s: (s.combatant.ac.total, s.hp, engine.states.index(s)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _melee_actions(
|
def _melee_actions(
|
||||||
|
|||||||
@@ -0,0 +1,249 @@
|
|||||||
|
"""Phase-2 adversary compilation tests (docs/phase-2-llm-architecture.md 10 P4.1).
|
||||||
|
|
||||||
|
Deterministic, offline: no network, no LLM. Pins the ``AdversarySpec`` /
|
||||||
|
``compile_adversary`` / ``compile_matchup`` contract (default-policy identity,
|
||||||
|
per-side routing on a real ``CombatEngine``, empty ``party_side`` rejection),
|
||||||
|
the additive ``"healer"`` target priority (prefer the enemy caster, fall back
|
||||||
|
to ``weakest_ac``), and ``SideSpec.aware`` landing on the right side's
|
||||||
|
combatants plus its SRD "Surprise" round effect. Conventions follow
|
||||||
|
tests/test_phase2_search.py: small n (runs <= 5), fixed seed, ROOT-relative
|
||||||
|
path constants.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from pf1e_simulator.adversary import (
|
||||||
|
AdversarySpec,
|
||||||
|
adversary_name,
|
||||||
|
compile_adversary,
|
||||||
|
compile_matchup,
|
||||||
|
)
|
||||||
|
from pf1e_simulator.combat import CombatantState, CombatEngine, default_policy
|
||||||
|
from pf1e_simulator.grid import Grid
|
||||||
|
from pf1e_simulator.loaders import load_combatant
|
||||||
|
from pf1e_simulator.map import load_map
|
||||||
|
from pf1e_simulator.rng import SeededRng
|
||||||
|
from pf1e_simulator.runner import build_states, run_encounter
|
||||||
|
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec, build_encounter
|
||||||
|
from pf1e_simulator.strategy import StrategySpec, compile_strategy, select_target
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pf1e_simulator.models import Combatant
|
||||||
|
from pf1e_simulator.runner import EncounterSpec
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||||
|
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
||||||
|
ORC = ROOT / "data" / "monsters" / "orc.json"
|
||||||
|
NAIRDA = ROOT / "data" / "pcs" / "nairda.json"
|
||||||
|
|
||||||
|
RUNS = 5
|
||||||
|
SEED = 7
|
||||||
|
ROUND_CAP = 20
|
||||||
|
PARTY = "players"
|
||||||
|
|
||||||
|
_MAP_SPEC = load_map(MAP)
|
||||||
|
|
||||||
|
|
||||||
|
def _c(path: Path) -> Combatant:
|
||||||
|
return load_combatant(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _scenario(*, monsters_aware: bool | None = None) -> ScenarioSpec:
|
||||||
|
"""3 goblins (zone A) vs 2 orcs (zone B), optionally with side awareness."""
|
||||||
|
return ScenarioSpec(
|
||||||
|
name="goblins-vs-orcs",
|
||||||
|
map_file=MAP,
|
||||||
|
sides=(
|
||||||
|
SideSpec(
|
||||||
|
name=PARTY,
|
||||||
|
zone="A",
|
||||||
|
combatant_files=(GOBLIN, GOBLIN, GOBLIN),
|
||||||
|
),
|
||||||
|
SideSpec(
|
||||||
|
name="monsters",
|
||||||
|
zone="B",
|
||||||
|
combatant_files=(ORC, ORC),
|
||||||
|
aware=monsters_aware,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
round_cap=ROUND_CAP,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _encounter() -> EncounterSpec:
|
||||||
|
return build_encounter(_scenario())
|
||||||
|
|
||||||
|
|
||||||
|
def _state(
|
||||||
|
combatant: Combatant, side: str, pos: tuple[int, int], *, hp: int | None = None
|
||||||
|
) -> CombatantState:
|
||||||
|
"""A bare runtime state over a loaded combatant (no battle run)."""
|
||||||
|
return CombatantState(
|
||||||
|
combatant=combatant,
|
||||||
|
side=side,
|
||||||
|
pos=pos,
|
||||||
|
hp=combatant.hp_max if hp is None else hp,
|
||||||
|
resources_left={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _mini_engine(states: list[CombatantState]) -> CombatEngine:
|
||||||
|
"""A live engine over the sample arena with the given states (no battle run)."""
|
||||||
|
return CombatEngine(SeededRng(SEED), Grid.from_spec(_MAP_SPEC), states)
|
||||||
|
|
||||||
|
|
||||||
|
# --- identity ---------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_adversary_default_is_default_policy() -> None:
|
||||||
|
"""The all-default AdversarySpec compiles to default_policy by identity."""
|
||||||
|
assert compile_adversary(AdversarySpec()) is default_policy
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_matchup_defaults_is_default_policy() -> None:
|
||||||
|
"""Two all-default specs return default_policy itself, not a wrapper."""
|
||||||
|
assert compile_matchup(StrategySpec(), AdversarySpec()) is default_policy
|
||||||
|
|
||||||
|
|
||||||
|
def test_adversary_name_mirrors_strategy_name() -> None:
|
||||||
|
"""Labels render non-default levers in field order; the name field is excluded."""
|
||||||
|
assert adversary_name(AdversarySpec()) == "default"
|
||||||
|
assert adversary_name(AdversarySpec(target_priority="healer")) == "healer"
|
||||||
|
assert (
|
||||||
|
adversary_name(AdversarySpec(name="custom", spell_policy="none"))
|
||||||
|
== "spell_policy=none"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
adversary_name(
|
||||||
|
AdversarySpec(target_priority="weakest_ac", retreat_hp_fraction=0.5)
|
||||||
|
)
|
||||||
|
== "weakest_ac, retreat=0.5"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- validation ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_matchup_empty_party_side_raises() -> None:
|
||||||
|
"""An empty party_side is a ValueError (pre-assigned msg, EM101/EM102)."""
|
||||||
|
with pytest.raises(ValueError, match="party_side"):
|
||||||
|
compile_matchup(StrategySpec(), AdversarySpec(), party_side="")
|
||||||
|
|
||||||
|
|
||||||
|
# --- routing ----------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_matchup_routes_by_side() -> None:
|
||||||
|
"""On a real engine, each side of the matchup delegates to its own policy."""
|
||||||
|
party_spec = StrategySpec(name="party-focus", target_priority="weakest_ac")
|
||||||
|
adversary_spec = AdversarySpec(name="adv-focus", target_priority="lowest_hp")
|
||||||
|
matchup = compile_matchup(party_spec, adversary_spec)
|
||||||
|
assert matchup is not default_policy
|
||||||
|
|
||||||
|
encounter = _encounter()
|
||||||
|
states = build_states(encounter)
|
||||||
|
engine = CombatEngine(SeededRng(SEED), Grid.from_spec(encounter.map), states)
|
||||||
|
party_state = next(s for s in states if s.side == PARTY)
|
||||||
|
monster_state = next(s for s in states if s.side == "monsters")
|
||||||
|
|
||||||
|
assert matchup(engine, party_state) == compile_strategy(party_spec)(
|
||||||
|
engine, party_state
|
||||||
|
)
|
||||||
|
assert matchup(engine, monster_state) == compile_adversary(adversary_spec)(
|
||||||
|
engine, monster_state
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_matchup_and_healer_strategy_run_deterministically() -> None:
|
||||||
|
"""A healer spec compiles and runs evaluate-style cells bit-identically."""
|
||||||
|
healer = StrategySpec(name="healer-party", target_priority="healer")
|
||||||
|
policy = compile_strategy(healer)
|
||||||
|
assert policy is not default_policy
|
||||||
|
matchup = compile_matchup(healer, AdversarySpec(target_priority="weakest_ac"))
|
||||||
|
|
||||||
|
spec = _encounter()
|
||||||
|
_, first = run_encounter(spec, runs=RUNS, seed=SEED, policy=policy, collect_runs=True)
|
||||||
|
_, again = run_encounter(spec, runs=RUNS, seed=SEED, policy=policy, collect_runs=True)
|
||||||
|
assert first == again
|
||||||
|
_, matchup_a = run_encounter(
|
||||||
|
spec, runs=RUNS, seed=SEED, policy=matchup, collect_runs=True
|
||||||
|
)
|
||||||
|
_, matchup_b = run_encounter(
|
||||||
|
spec, runs=RUNS, seed=SEED, policy=matchup, collect_runs=True
|
||||||
|
)
|
||||||
|
assert matchup_a == matchup_b
|
||||||
|
|
||||||
|
|
||||||
|
# --- healer targeting -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_healer_prefers_the_enemy_caster() -> None:
|
||||||
|
"""With a monster-side caster (nairda) among non-casters, healer picks her."""
|
||||||
|
actor = _state(_c(GOBLIN), PARTY, (1, 1))
|
||||||
|
nairda = _state(_c(NAIRDA), "monsters", (1, 8))
|
||||||
|
orc = _state(_c(ORC), "monsters", (1, 16))
|
||||||
|
assert nairda.combatant.spells # caster precondition (nairda.json)
|
||||||
|
engine = _mini_engine([actor, nairda, orc])
|
||||||
|
|
||||||
|
assert select_target(engine, actor, "healer") is nairda
|
||||||
|
# Discriminator: weakest_ac sees equal AC 13 and prefers the wounded orc.
|
||||||
|
assert select_target(engine, actor, "weakest_ac") is orc
|
||||||
|
|
||||||
|
|
||||||
|
def test_healer_falls_back_to_weakest_ac_without_casters() -> None:
|
||||||
|
"""No active caster on the enemy side -> identical to weakest_ac."""
|
||||||
|
actor = _state(_c(GOBLIN), PARTY, (1, 1))
|
||||||
|
orc = _state(_c(ORC), "monsters", (1, 8))
|
||||||
|
goblin_enemy = _state(_c(GOBLIN), "monsters", (1, 16))
|
||||||
|
assert not orc.combatant.spells
|
||||||
|
assert not goblin_enemy.combatant.spells
|
||||||
|
engine = _mini_engine([actor, orc, goblin_enemy])
|
||||||
|
|
||||||
|
healer = select_target(engine, actor, "healer")
|
||||||
|
assert healer is select_target(engine, actor, "weakest_ac")
|
||||||
|
assert healer is orc # AC 13 < goblin AC 16
|
||||||
|
|
||||||
|
|
||||||
|
# --- SideSpec.aware (SRD "Surprise") ---------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_side_spec_aware_applies_per_side() -> None:
|
||||||
|
"""aware=False lands on that side's combatants only; the other side keeps True."""
|
||||||
|
encounter = build_encounter(_scenario(monsters_aware=False))
|
||||||
|
players, monsters = encounter.sides
|
||||||
|
assert all(c.aware is True for c in players.combatants)
|
||||||
|
assert all(c.aware is False for c in monsters.combatants)
|
||||||
|
|
||||||
|
states = build_states(encounter)
|
||||||
|
assert all(s.combatant.aware is True for s in states if s.side == PARTY)
|
||||||
|
assert all(s.combatant.aware is False for s in states if s.side == "monsters")
|
||||||
|
|
||||||
|
untouched = build_encounter(_scenario())
|
||||||
|
assert all(c.aware is True for side in untouched.sides for c in side.combatants)
|
||||||
|
|
||||||
|
|
||||||
|
def test_side_spec_aware_triggers_surprise_round() -> None:
|
||||||
|
"""Only some aware -> a surprise round runs; the default (aware=None) has none."""
|
||||||
|
surprise = _encounter_states_and_run(_scenario(monsters_aware=False))
|
||||||
|
assert "surprise round" in surprise
|
||||||
|
|
||||||
|
default = _encounter_states_and_run(_scenario())
|
||||||
|
assert "surprise round" not in default
|
||||||
|
|
||||||
|
|
||||||
|
def _encounter_states_and_run(scenario: ScenarioSpec) -> tuple[str, ...]:
|
||||||
|
"""Run one seeded battle of `scenario` and return its transcript."""
|
||||||
|
encounter = build_encounter(scenario)
|
||||||
|
engine = CombatEngine(
|
||||||
|
SeededRng(SEED),
|
||||||
|
Grid.from_spec(encounter.map),
|
||||||
|
build_states(encounter),
|
||||||
|
round_cap=ROUND_CAP,
|
||||||
|
)
|
||||||
|
return engine.run().transcript
|
||||||
@@ -0,0 +1,599 @@
|
|||||||
|
"""Tests for the robustness layer (P4.2/P4.3, docs section 10)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from pf1e_simulator import robustness
|
||||||
|
from pf1e_simulator.adversary import AdversarySpec
|
||||||
|
from pf1e_simulator.robustness import (
|
||||||
|
DangerCell,
|
||||||
|
MatchupBenchmark,
|
||||||
|
RobustnessEntry,
|
||||||
|
SensitivityEntry,
|
||||||
|
adversarial_analysis,
|
||||||
|
danger_map,
|
||||||
|
format_danger_map,
|
||||||
|
format_frontier,
|
||||||
|
format_matchup,
|
||||||
|
format_robustness,
|
||||||
|
format_sensitivity,
|
||||||
|
frontier,
|
||||||
|
matchup_benchmark,
|
||||||
|
sensitivity,
|
||||||
|
)
|
||||||
|
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
|
||||||
|
from pf1e_simulator.strategy import StrategySpec
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
RUNS = 5
|
||||||
|
SEED = 7
|
||||||
|
ROUND_CAP = 20
|
||||||
|
PARTY = "players"
|
||||||
|
|
||||||
|
ARENA = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||||
|
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
||||||
|
ORC = ROOT / "data" / "monsters" / "orc.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _scenario(
|
||||||
|
*, name: str = "goblins-vs-orcs", sides: tuple[SideSpec, ...] | None = None
|
||||||
|
) -> ScenarioSpec:
|
||||||
|
return ScenarioSpec(
|
||||||
|
name=name,
|
||||||
|
map_file=ARENA,
|
||||||
|
sides=sides
|
||||||
|
or (
|
||||||
|
SideSpec(PARTY, "A", (GOBLIN, GOBLIN, GOBLIN)),
|
||||||
|
SideSpec("monsters", "B", (ORC, ORC)),
|
||||||
|
),
|
||||||
|
round_cap=ROUND_CAP,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _party(*, name: str = "default") -> StrategySpec:
|
||||||
|
return StrategySpec(name=name)
|
||||||
|
|
||||||
|
|
||||||
|
def _three_side_scenario() -> ScenarioSpec:
|
||||||
|
return ScenarioSpec(
|
||||||
|
name="three-sides",
|
||||||
|
map_file=ARENA,
|
||||||
|
sides=(
|
||||||
|
SideSpec(PARTY, "A", (GOBLIN,)),
|
||||||
|
SideSpec("monsters", "B", (ORC,)),
|
||||||
|
SideSpec("spectators", "C", (GOBLIN,)),
|
||||||
|
),
|
||||||
|
round_cap=ROUND_CAP,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_matchup_requires_exactly_two_sides() -> None:
|
||||||
|
with pytest.raises(ValueError, match="exactly 2 sides"):
|
||||||
|
matchup_benchmark(
|
||||||
|
_three_side_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_matchup_requires_valid_party_side() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
with pytest.raises(ValueError, match="party_side"):
|
||||||
|
matchup_benchmark(
|
||||||
|
scenario, _party(), AdversarySpec(), party_side="", runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="is not a side"):
|
||||||
|
matchup_benchmark(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
party_side="spectators",
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_matchup_reports_both_dangers_and_is_reproducible() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
party = _party()
|
||||||
|
adversary = AdversarySpec()
|
||||||
|
first = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
|
||||||
|
assert isinstance(first, MatchupBenchmark)
|
||||||
|
assert first.scenario == "goblins-vs-orcs"
|
||||||
|
assert first.party == "default"
|
||||||
|
assert first.adversary == "default"
|
||||||
|
assert first.runs == RUNS
|
||||||
|
assert first.seed == SEED
|
||||||
|
assert first.party_side == PARTY
|
||||||
|
assert first.report.runs == RUNS
|
||||||
|
for danger in (first.party_danger, first.adversary_danger):
|
||||||
|
assert 0.0 <= danger.p_survives <= 1.0
|
||||||
|
assert 0.0 <= danger.p_tpk <= 1.0
|
||||||
|
assert 0.0 <= danger.p_any_death <= 1.0
|
||||||
|
assert danger.expected_hp_lost >= 0.0
|
||||||
|
second = matchup_benchmark(scenario, party, adversary, runs=RUNS, seed=SEED)
|
||||||
|
assert first == second
|
||||||
|
assert second.party_danger == first.party_danger
|
||||||
|
|
||||||
|
|
||||||
|
def test_adversarial_analysis_alternates_and_converges() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
party = _party()
|
||||||
|
specs = (
|
||||||
|
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
|
||||||
|
AdversarySpec(name="plain"),
|
||||||
|
)
|
||||||
|
entries = adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=[party],
|
||||||
|
adversary_candidates=specs,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
iterations=2,
|
||||||
|
)
|
||||||
|
assert len(entries) == 2
|
||||||
|
assert all(isinstance(entry, RobustnessEntry) for entry in entries)
|
||||||
|
assert [entry.iteration for entry in entries] == [1, 2]
|
||||||
|
# the adversary best response is the candidate with the worst party outlook
|
||||||
|
benchmarks = {
|
||||||
|
spec.name: matchup_benchmark(
|
||||||
|
scenario, party, spec, runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
for spec in specs
|
||||||
|
}
|
||||||
|
expected = min(
|
||||||
|
benchmarks.values(),
|
||||||
|
key=lambda b: (
|
||||||
|
b.party_danger.p_survives,
|
||||||
|
-b.party_danger.expected_hp_lost,
|
||||||
|
b.adversary,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert all(entry.benchmark == expected for entry in entries)
|
||||||
|
assert entries[0].adversary.name == expected.adversary
|
||||||
|
|
||||||
|
|
||||||
|
def test_adversarial_party_best_response() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
parties = (
|
||||||
|
StrategySpec(),
|
||||||
|
StrategySpec(name="bully", target_priority="lowest_hp"),
|
||||||
|
)
|
||||||
|
adversary = AdversarySpec()
|
||||||
|
entries = adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=parties,
|
||||||
|
adversary_candidates=[adversary],
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
iterations=1,
|
||||||
|
)
|
||||||
|
benchmarks = {
|
||||||
|
spec.name: matchup_benchmark(
|
||||||
|
scenario, spec, adversary, runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
for spec in parties
|
||||||
|
}
|
||||||
|
expected = max(
|
||||||
|
benchmarks.values(),
|
||||||
|
key=lambda b: (
|
||||||
|
b.party_danger.p_survives,
|
||||||
|
-b.party_danger.expected_hp_lost,
|
||||||
|
b.party,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert entries[0].benchmark == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_adversarial_candidates_required() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
party = _party()
|
||||||
|
with pytest.raises(ValueError, match="party_candidates must be non-empty"):
|
||||||
|
adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=[],
|
||||||
|
adversary_candidates=[AdversarySpec()],
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="adversary_candidates must be non-empty"):
|
||||||
|
adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=[party],
|
||||||
|
adversary_candidates=[],
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_adversarial_budget_is_enforced_before_any_battle() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
party = _party()
|
||||||
|
specs = (
|
||||||
|
AdversarySpec(name="retreat", retreat_hp_fraction=0.5),
|
||||||
|
AdversarySpec(name="plain"),
|
||||||
|
)
|
||||||
|
cost = 2 * (1 + len(specs)) * RUNS
|
||||||
|
with pytest.raises(ValueError, match="robustness budget exceeded"):
|
||||||
|
adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=[party],
|
||||||
|
adversary_candidates=specs,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
iterations=2,
|
||||||
|
max_total_runs=cost - 1,
|
||||||
|
)
|
||||||
|
entries = adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=[party],
|
||||||
|
adversary_candidates=specs,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
iterations=2,
|
||||||
|
max_total_runs=cost,
|
||||||
|
)
|
||||||
|
assert len(entries) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_danger_map_is_3x3_party_major_with_variant_names() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
cells = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||||
|
assert len(cells) == 9
|
||||||
|
assert all(isinstance(cell, DangerCell) for cell in cells)
|
||||||
|
order = [(cell.party_hp_fraction, cell.adversary_hp_fraction) for cell in cells]
|
||||||
|
assert order == [(p, a) for p in (0.5, 0.7, 1.0) for a in (0.5, 0.7, 1.0)]
|
||||||
|
assert cells[0].benchmark.scenario == "goblins-vs-orcs party_hp=0.5 adv_hp=0.5"
|
||||||
|
assert cells[-1].benchmark.scenario == "goblins-vs-orcs party_hp=1.0 adv_hp=1.0"
|
||||||
|
for cell in cells:
|
||||||
|
assert 0.0 <= cell.benchmark.party_danger.p_survives <= 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_danger_map_rejects_out_of_range_fractions() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
with pytest.raises(ValueError, match="fraction must be in"):
|
||||||
|
danger_map(
|
||||||
|
scenario, _party(), AdversarySpec(), fractions=(0.0,), runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="fraction must be in"):
|
||||||
|
danger_map(
|
||||||
|
scenario, _party(), AdversarySpec(), fractions=(1.5,), runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="fractions must be non-empty"):
|
||||||
|
danger_map(
|
||||||
|
scenario, _party(), AdversarySpec(), fractions=(), runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_danger_map_is_reproducible() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
first = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||||
|
second = danger_map(scenario, _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||||
|
assert first == second
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_finds_reached_threshold() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
result = frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
min_fraction=0.25,
|
||||||
|
max_fraction=1.0,
|
||||||
|
precision=0.05,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
# expected hp lost is always >= 0, so the threshold is crossed everywhere:
|
||||||
|
# bisection converges to the smallest probed fraction, rounded to precision
|
||||||
|
assert result == pytest.approx(0.25)
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_returns_none_when_threshold_unreachable() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
result = frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=1e9,
|
||||||
|
min_fraction=0.25,
|
||||||
|
max_fraction=1.0,
|
||||||
|
precision=0.05,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_is_reproducible() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
first = frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
second = frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
assert first == second
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_returns_precision_multiple() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
result = frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
min_fraction=0.3,
|
||||||
|
max_fraction=1.0,
|
||||||
|
precision=0.1,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
# bisection stops within precision of min_fraction; the result is the
|
||||||
|
# nearest precision multiple in [min_fraction, min_fraction + precision]
|
||||||
|
assert 0.3 <= result <= 0.4
|
||||||
|
assert result == pytest.approx(round(result / 0.1) * 0.1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_p_survives_uses_inverse_threshold() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
result = frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="p_survives",
|
||||||
|
target=0.9999,
|
||||||
|
min_fraction=0.5,
|
||||||
|
max_fraction=1.0,
|
||||||
|
precision=0.25,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
# convention: p_survives crosses when value <= target; a present result
|
||||||
|
# is a precision multiple within the probed range
|
||||||
|
if result is not None:
|
||||||
|
assert 0.5 <= result <= 0.75
|
||||||
|
assert result == pytest.approx(round(result / 0.25) * 0.25)
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_validates_arguments() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
with pytest.raises(ValueError, match="criterion must be one of"):
|
||||||
|
frontier(
|
||||||
|
scenario, _party(), AdversarySpec(), criterion="bogus", runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="target must be in"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="p_tpk",
|
||||||
|
target=0.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="target must be in"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="p_survives",
|
||||||
|
target=1.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="target must be >= 0"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=-1.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="min_fraction must be > 0"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
min_fraction=0.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="max_fraction must be <= 1"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
max_fraction=1.5,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="min_fraction must be <= max_fraction"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
min_fraction=0.9,
|
||||||
|
max_fraction=0.5,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="precision must be > 0"):
|
||||||
|
frontier(
|
||||||
|
scenario,
|
||||||
|
_party(),
|
||||||
|
AdversarySpec(),
|
||||||
|
criterion="expected_hp_lost",
|
||||||
|
target=0.0,
|
||||||
|
precision=0.0,
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sensitivity_ranks_levers_and_reports_signed_deltas() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
party = _party()
|
||||||
|
entries = sensitivity(scenario, party, runs=RUNS, seed=SEED)
|
||||||
|
assert len(entries) == 7
|
||||||
|
assert all(isinstance(entry, SensitivityEntry) for entry in entries)
|
||||||
|
assert {entry.levers for entry in entries} == {
|
||||||
|
"weakest_ac",
|
||||||
|
"lowest_hp",
|
||||||
|
"healer",
|
||||||
|
"spell_policy=none",
|
||||||
|
"retreat=0.25",
|
||||||
|
"retreat=0.5",
|
||||||
|
"use_charges=false",
|
||||||
|
}
|
||||||
|
for previous, current in itertools.pairwise(entries):
|
||||||
|
assert abs(previous.delta) >= abs(current.delta)
|
||||||
|
if abs(previous.delta) == abs(current.delta):
|
||||||
|
assert previous.levers <= current.levers
|
||||||
|
# delta is the signed change vs the all-default baseline
|
||||||
|
baseline = matchup_benchmark(
|
||||||
|
scenario, party, AdversarySpec(), runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
base = baseline.party_danger.expected_hp_lost
|
||||||
|
for entry in entries:
|
||||||
|
assert entry.delta == pytest.approx(
|
||||||
|
entry.benchmark.party_danger.expected_hp_lost - base
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sensitivity_rejects_unknown_criterion() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
with pytest.raises(ValueError, match="criterion must be one of"):
|
||||||
|
sensitivity(scenario, _party(), criterion="bogus", runs=RUNS, seed=SEED)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sensitivity_budget_is_enforced() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
party = _party()
|
||||||
|
with pytest.raises(ValueError, match="robustness budget exceeded"):
|
||||||
|
sensitivity(
|
||||||
|
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS - 1
|
||||||
|
)
|
||||||
|
entries = sensitivity(
|
||||||
|
scenario, party, runs=RUNS, seed=SEED, max_total_runs=8 * RUNS
|
||||||
|
)
|
||||||
|
assert len(entries) == 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_matchup_ascii_aligned() -> None:
|
||||||
|
benchmark_ = matchup_benchmark(
|
||||||
|
_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED
|
||||||
|
)
|
||||||
|
text = format_matchup(benchmark_)
|
||||||
|
lines = text.splitlines()
|
||||||
|
assert lines[0].split() == [
|
||||||
|
"Scenario",
|
||||||
|
"Runs",
|
||||||
|
"Side",
|
||||||
|
"Win%",
|
||||||
|
"Survive",
|
||||||
|
"TPK",
|
||||||
|
">=1-death",
|
||||||
|
"HP-lost",
|
||||||
|
"Rounds",
|
||||||
|
]
|
||||||
|
assert lines[1] == "-" * len(lines[0])
|
||||||
|
assert "party" in lines[2]
|
||||||
|
assert "adversary" in lines[3]
|
||||||
|
assert all(len(line) == len(lines[0]) for line in lines)
|
||||||
|
assert text.isascii()
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_robustness_ascii_aligned() -> None:
|
||||||
|
scenario = _scenario()
|
||||||
|
entries = adversarial_analysis(
|
||||||
|
scenario,
|
||||||
|
party_candidates=[_party()],
|
||||||
|
adversary_candidates=[AdversarySpec()],
|
||||||
|
runs=RUNS,
|
||||||
|
seed=SEED,
|
||||||
|
iterations=2,
|
||||||
|
)
|
||||||
|
text = format_robustness(entries)
|
||||||
|
lines = text.splitlines()
|
||||||
|
assert lines[0].split() == [
|
||||||
|
"Iter",
|
||||||
|
"Party",
|
||||||
|
"Adversary",
|
||||||
|
"P(party-survive)",
|
||||||
|
"P(party-TPK)",
|
||||||
|
"Party-HP",
|
||||||
|
"P(adv-survive)",
|
||||||
|
"P(adv-TPK)",
|
||||||
|
"Adv-HP",
|
||||||
|
]
|
||||||
|
assert len(lines) == 2 + 2
|
||||||
|
assert all(len(line) == len(lines[0]) for line in lines)
|
||||||
|
assert text.isascii()
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_danger_map_ascii_aligned() -> None:
|
||||||
|
cells = danger_map(_scenario(), _party(), AdversarySpec(), runs=RUNS, seed=SEED)
|
||||||
|
text = format_danger_map(cells)
|
||||||
|
lines = text.splitlines()
|
||||||
|
assert lines[0].startswith("Party\\Adv")
|
||||||
|
assert "adv=0.5" in lines[0]
|
||||||
|
assert "party=0.5" in lines[2]
|
||||||
|
assert "party=1.0" in lines[4]
|
||||||
|
assert "/" in lines[2]
|
||||||
|
assert all(len(line) == len(lines[0]) for line in lines)
|
||||||
|
assert text.isascii()
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_frontier_messages() -> None:
|
||||||
|
assert (
|
||||||
|
format_frontier("p_tpk", 0.2, None, min_fraction=0.25, max_fraction=1.0)
|
||||||
|
== "no frontier for p_tpk 0.2 in [0.25, 1.00]"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
format_frontier("p_tpk", 0.2, 0.35, min_fraction=0.25, max_fraction=1.0)
|
||||||
|
== "frontier p_tpk 0.2: party hp_fraction = 0.35"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_sensitivity_ascii_aligned() -> None:
|
||||||
|
entries = sensitivity(_scenario(), _party(), runs=RUNS, seed=SEED)
|
||||||
|
text = format_sensitivity(entries, "expected_hp_lost")
|
||||||
|
lines = text.splitlines()
|
||||||
|
assert lines[0].split() == ["Rank", "Levers", "Value", "Delta"]
|
||||||
|
assert len(lines) == 2 + 7
|
||||||
|
assert lines[1].split()[-1].startswith(("+", "-"))
|
||||||
|
assert all(len(line) == len(lines[0]) for line in lines)
|
||||||
|
assert text.isascii()
|
||||||
|
|
||||||
|
|
||||||
|
def test_robustness_module_does_not_import_search() -> None:
|
||||||
|
source = Path(robustness.__file__).resolve().read_text(encoding="utf-8")
|
||||||
|
assert "search" not in source
|
||||||
Reference in New Issue
Block a user