feat(phase2): danger-cartography spike - per-run outcomes, objectives, focus-fire adversary
run_encounter gains an opt-in collect_runs flag returning one RunOutcome per battle (winner, rounds, per-combatant id/side/hp/hp_max/dead) next to the EncounterReport; default behavior is unchanged. objectives.py provides the pure optimizer primitives: P(survives all), P(TPK), P(>=1 death), expected hp lost, expected rounds and a worst-decile (CVaR-style) tail metric. phase2_spike.py holds the spike-level strengthened monster policy (focus fire on the lowest-AC enemy, ties to the wounded; party side stays on default_policy). tests/test_phase2_danger_map.py prints the danger-map sketch: warband and lich scenarios measured default vs focus-fire at 100 runs / seed 11, pinning the delta deterministically.
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
"""Unit tests for the Phase-2 objective primitives (objectives.py).
|
||||
|
||||
Synthetic RunOutcome tuples with hand-computed expectations: the metrics
|
||||
are pure frequencies/means over per-run summaries, so every number below is
|
||||
verifiable by eye. No engine, no RNG.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pf1e_simulator.objectives import (
|
||||
danger_metrics,
|
||||
expected_hp_lost,
|
||||
expected_rounds,
|
||||
p_any_death,
|
||||
p_survives,
|
||||
p_tpk,
|
||||
worst_decile_party_hp,
|
||||
)
|
||||
from pf1e_simulator.runner import CombatantOutcome, RunOutcome
|
||||
|
||||
|
||||
def _pc(cid: str, hp: int, hp_max: int = 10) -> CombatantOutcome:
|
||||
dead = hp <= -12
|
||||
return CombatantOutcome(id=cid, side="players", hp=hp, hp_max=hp_max, dead=dead)
|
||||
|
||||
|
||||
def _mob(cid: str, hp: int, hp_max: int = 10) -> CombatantOutcome:
|
||||
dead = hp <= -12
|
||||
return CombatantOutcome(id=cid, side="monsters", hp=hp, hp_max=hp_max, dead=dead)
|
||||
|
||||
|
||||
def _run(rounds: int, *combatants: CombatantOutcome) -> RunOutcome:
|
||||
return RunOutcome(winner=None, rounds=rounds, combatants=combatants)
|
||||
|
||||
|
||||
def test_empty_batch_returns_zero() -> None:
|
||||
assert p_survives((), "players") == 0.0
|
||||
assert p_tpk((), "players") == 0.0
|
||||
assert p_any_death((), "players") == 0.0
|
||||
assert expected_hp_lost((), "players") == 0.0
|
||||
assert expected_rounds(()) == 0.0
|
||||
assert worst_decile_party_hp((), "players") == 0.0
|
||||
|
||||
|
||||
def test_survival_tpk_and_death_rates() -> None:
|
||||
# run 1: both players clean (hp 10/10, 8/10) -> survives, no death
|
||||
# run 2: one player dying (hp -1, NOT dead at -CON 12), one at 3 hp
|
||||
# -> survives (nobody dead), no death counted
|
||||
# run 3: one player dead (hp -12), one at 0 hp -> any death, not TPK
|
||||
# run 4: both dead (hp -13, -12) -> TPK
|
||||
runs = (
|
||||
_run(2, _pc("a", 10), _pc("b", 8), _mob("m", 5)),
|
||||
_run(3, _pc("a", -1), _pc("b", 3)),
|
||||
_run(4, _pc("a", -12), _pc("b", 0)),
|
||||
_run(5, _pc("a", -13), _pc("b", -12)),
|
||||
)
|
||||
assert p_survives(runs, "players") == 2 / 4
|
||||
assert p_any_death(runs, "players") == 2 / 4
|
||||
assert p_tpk(runs, "players") == 1 / 4
|
||||
# monsters absent from a run: vacuously survived, never TPK
|
||||
assert p_survives(runs, "monsters") == 1.0
|
||||
assert p_tpk(runs, "monsters") == 0.0
|
||||
|
||||
|
||||
def test_expected_hp_lost_floors_per_member() -> None:
|
||||
# run 1: a lost 2, b lost 7 (total 9). Run 2: a at -12 counts as the
|
||||
# full 10 lost (clamped), b at 10 lost nothing. Mean of the two totals
|
||||
# is 9.5.
|
||||
runs = (
|
||||
_run(1, _pc("a", 8), _pc("b", 3)),
|
||||
_run(2, _pc("a", -12), _pc("b", 10)),
|
||||
)
|
||||
assert expected_hp_lost(runs, "players") == 9.5
|
||||
assert expected_rounds(runs) == 1.5
|
||||
|
||||
|
||||
def test_worst_decile_takes_the_tail_not_the_mean() -> None:
|
||||
# 20 runs: 19 clean (hp lost 0) + 1 disaster (both dead, 20 lost).
|
||||
# Mean hp lost = 1.0 but the worst decile (2 runs: disaster + one clean)
|
||||
# averages 10 hp remaining - the tail is visible.
|
||||
clean = _run(1, _pc("a", 10), _pc("b", 10))
|
||||
disaster = _run(9, _pc("a", -12), _pc("b", -13))
|
||||
runs = (disaster, *([clean] * 19))
|
||||
assert expected_hp_lost(runs, "players") == 1.0
|
||||
assert worst_decile_party_hp(runs, "players") == (0 + 20) / 2
|
||||
|
||||
|
||||
def test_worst_decile_minimum_one_run() -> None:
|
||||
runs = (_run(1, _pc("a", 4)),)
|
||||
assert worst_decile_party_hp(runs, "players") == 4
|
||||
|
||||
|
||||
def test_danger_metrics_aggregates_every_primitive() -> None:
|
||||
runs = (
|
||||
_run(2, _pc("a", 10), _mob("m", 0)),
|
||||
_run(4, _pc("a", -12), _mob("m", -13)),
|
||||
)
|
||||
m = danger_metrics(runs, "players")
|
||||
assert m.runs == 2
|
||||
assert m.p_survives == 0.5
|
||||
assert m.p_tpk == 0.5
|
||||
assert m.p_any_death == 0.5
|
||||
assert m.expected_hp_lost == 5.0
|
||||
assert m.expected_rounds == 3.0
|
||||
assert m.worst_decile_party_hp == 0.0
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Phase-2 danger-cartography spike (docs/phase-2-llm-architecture.md 3.1/3.2, 10 P3/P4).
|
||||
|
||||
Proof of value WITHOUT any LLM: the same party measured against the default
|
||||
monster policy and against the strengthened focus-fire policy (spike module
|
||||
pf1e_simulator.phase2_spike). A danger only exists against an adversary that
|
||||
plays to win, so the DELTA between the two columns is the danger the default
|
||||
Monte Carlo report hides. Prints a compact per-scenario block (party win
|
||||
rate +/- 3 sigma, P(TPK), P(>=1 death), expected hp lost, rounds, worst
|
||||
decile hp) and pins loose assertions: everything is deterministic under a
|
||||
fixed seed, so the pinned inequalities are exact reproductions, not noise.
|
||||
|
||||
Scenarios (sample arena, 100 runs, fixed seed):
|
||||
warband - 4 level-4 PCs vs a thrune agent + 2 orcs + a goblin: the party
|
||||
"wins" either way, but focus fire nearly doubles P(>=1 death).
|
||||
lich - 2 level-2/4 PCs vs the lich + an orc: the party loses both
|
||||
ways, but against the default policy nobody ever actually
|
||||
DIES (downed at hp<0, above -CON); focus fire converts defeat
|
||||
into real deaths and a nonzero TPK rate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pf1e_simulator.combat import CombatantState, CombatEngine, default_policy
|
||||
from pf1e_simulator.grid import Grid
|
||||
from pf1e_simulator.loaders import load_combatant
|
||||
from pf1e_simulator.map import load_map
|
||||
from pf1e_simulator.objectives import DangerMetrics, danger_metrics
|
||||
from pf1e_simulator.phase2_spike import focus_fire_policy
|
||||
from pf1e_simulator.rng import SeededRng
|
||||
from pf1e_simulator.runner import EncounterSpec, RunOutcome, Side, run_encounter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.models import Combatant
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
|
||||
RUNS = 100
|
||||
SEED = 11
|
||||
ROUND_CAP = 20
|
||||
|
||||
|
||||
def _c(path: str) -> Combatant:
|
||||
return load_combatant(ROOT / "data" / path)
|
||||
|
||||
|
||||
def _spec(pcs: list[str], monsters: list[str]) -> EncounterSpec:
|
||||
return EncounterSpec(
|
||||
map=load_map(MAP),
|
||||
round_cap=ROUND_CAP,
|
||||
sides=(
|
||||
Side(name="players", combatants=tuple(_c(p) for p in pcs), zone="A"),
|
||||
Side(name="monsters", combatants=tuple(_c(p) for p in monsters), zone="B"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
WARBAND = _spec(
|
||||
["pcs/jeanne.json", "pcs/tammara.json", "pcs/harvie.json", "pcs/ierlieth.json"],
|
||||
["monsters/thrune_agent.json", "monsters/orc.json", "monsters/orc.json",
|
||||
"monsters/goblin.json"],
|
||||
)
|
||||
LICH = _spec(
|
||||
["pcs/harvie.json", "pcs/ierlieth.json"],
|
||||
["monsters/lich.json", "monsters/orc.json"],
|
||||
)
|
||||
|
||||
|
||||
def _emit(line: str) -> None:
|
||||
sys.stdout.write(line + "\n")
|
||||
|
||||
|
||||
def _mini_engine(states: list[CombatantState]) -> CombatEngine:
|
||||
return CombatEngine(SeededRng(1), Grid.from_spec(WARBAND.map), states)
|
||||
|
||||
|
||||
def _state(cid: str, side: str, pos: tuple[int, int], hp: int = 10) -> CombatantState:
|
||||
c = _c("monsters/orc.json").model_copy(update={"id": cid})
|
||||
return CombatantState(combatant=c, side=side, pos=pos, hp=hp, resources_left={})
|
||||
|
||||
|
||||
def test_focus_fire_concentrates_on_lowest_ac_not_nearest() -> None:
|
||||
"""The monster ignores the adjacent tough target and converges on the squishy one."""
|
||||
policy = focus_fire_policy("monsters")
|
||||
orc = _state("orc1", "monsters", (1, 1))
|
||||
tough = _c("pcs/jeanne.json") # AC 23, adjacent to the orc
|
||||
squishy = _c("pcs/nairda.json") # AC 13, far away
|
||||
tough_state = CombatantState(
|
||||
combatant=tough, side="players", pos=(1, 2), hp=tough.hp_max, resources_left={}
|
||||
)
|
||||
squishy_state = CombatantState(
|
||||
combatant=squishy, side="players", pos=(1, 16), hp=squishy.hp_max, resources_left={}
|
||||
)
|
||||
engine = _mini_engine([orc, tough_state, squishy_state])
|
||||
actions = policy(engine, orc)
|
||||
assert all(a.target_id == "nairda" for a in actions), actions
|
||||
|
||||
|
||||
def test_focus_fire_tie_break_prefers_wounded() -> None:
|
||||
"""Equal AC: the monster focuses the member with the lower current hp."""
|
||||
policy = focus_fire_policy("monsters")
|
||||
orc = _state("orc1", "monsters", (1, 1))
|
||||
fresh = _c("monsters/goblin.json").model_copy(update={"id": "fresh"})
|
||||
hurt = _c("monsters/goblin.json").model_copy(update={"id": "hurt"})
|
||||
fresh_state = CombatantState(
|
||||
combatant=fresh, side="players", pos=(5, 5), hp=fresh.hp_max, resources_left={}
|
||||
)
|
||||
hurt_state = CombatantState(
|
||||
combatant=hurt, side="players", pos=(5, 10), hp=1, resources_left={}
|
||||
)
|
||||
engine = _mini_engine([orc, fresh_state, hurt_state])
|
||||
actions = policy(engine, orc)
|
||||
assert all(a.target_id == "hurt" for a in actions), actions
|
||||
|
||||
|
||||
def test_focus_fire_leaves_the_party_side_on_default_policy() -> None:
|
||||
"""Only the monster side changes: a party-side turn gets default_policy verbatim."""
|
||||
policy = focus_fire_policy("monsters")
|
||||
pc = _state("p", "players", (1, 1))
|
||||
mob = _state("m", "monsters", (1, 8))
|
||||
engine = _mini_engine([pc, mob])
|
||||
assert policy(engine, pc) == default_policy(engine, pc)
|
||||
|
||||
|
||||
def _report(name: str, m: DangerMetrics, win: float, sigma: float) -> None:
|
||||
_emit(
|
||||
f" {name:<9} win={win:6.1%} +/-{3 * sigma:4.1%} surv={m.p_survives:5.1%} "
|
||||
f"tpk={m.p_tpk:5.1%} >=1-death={m.p_any_death:5.1%} "
|
||||
f"hp-lost={m.expected_hp_lost:6.1f} rounds={m.expected_rounds:4.1f} "
|
||||
f"tail-hp={m.worst_decile_party_hp:5.1f}"
|
||||
)
|
||||
|
||||
|
||||
def _measure(
|
||||
spec: EncounterSpec,
|
||||
*,
|
||||
strengthened: bool
|
||||
) -> tuple[float, float, DangerMetrics, tuple[RunOutcome, ...]]:
|
||||
policy = focus_fire_policy("monsters") if strengthened else None
|
||||
report, outcomes = run_encounter(
|
||||
spec, runs=RUNS, seed=SEED, policy=policy, collect_runs=True
|
||||
)
|
||||
win = report.win_rate("players")
|
||||
return win, report.win_rate_sigma("players"), danger_metrics(outcomes, "players"), outcomes
|
||||
|
||||
|
||||
def test_danger_map_default_vs_strengthened(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Print the danger map sketch and pin the delta (deterministic, fixed seed)."""
|
||||
_emit("=== PHASE2 SUMMARY (danger cartography spike) ===")
|
||||
deltas: dict[str, tuple[float, float]] = {}
|
||||
metrics: dict[str, tuple[DangerMetrics, DangerMetrics]] = {}
|
||||
for name, spec in (("warband", WARBAND), ("lich", LICH)):
|
||||
_emit(f"scenario {name}: {len(spec.sides[0].combatants)} PCs vs "
|
||||
f"{len(spec.sides[1].combatants)} monsters, runs={RUNS}, seed={SEED}")
|
||||
w0, s0, m0, _ = _measure(spec, strengthened=False)
|
||||
w1, s1, m1, _ = _measure(spec, strengthened=True)
|
||||
_report("default", m0, w0, s0)
|
||||
_report("focusfire", m1, w1, s1)
|
||||
_emit(
|
||||
f" delta death-rate {m1.p_any_death - m0.p_any_death:+.1%} "
|
||||
f"hp-lost {m1.expected_hp_lost - m0.expected_hp_lost:+.1f}"
|
||||
)
|
||||
deltas[name] = (m0.p_any_death, m1.p_any_death)
|
||||
metrics[name] = (m0, m1)
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "PHASE2 SUMMARY" in out
|
||||
|
||||
# Loose, deterministic pins: the strengthened adversary never hides danger.
|
||||
for m0, m1 in metrics.values():
|
||||
assert 0.0 <= m1.p_any_death <= 1.0
|
||||
assert m1.p_tpk >= m0.p_tpk - 0.05
|
||||
assert m1.p_survives <= m0.p_survives + 0.05
|
||||
# warband: focus fire must strictly raise the death rate
|
||||
d0, d1 = deltas["warband"]
|
||||
assert d1 > d0
|
||||
# lich: default policy shows ZERO real deaths; focus fire shows some
|
||||
assert metrics["lich"][0].p_any_death == 0.0
|
||||
assert metrics["lich"][1].p_any_death > 0.0
|
||||
assert metrics["lich"][1].p_tpk > 0.0
|
||||
|
||||
|
||||
def test_danger_map_is_reproducible() -> None:
|
||||
"""Same seed, same policy -> byte-identical per-run outcome tuples."""
|
||||
a = _measure(LICH, strengthened=True)[3]
|
||||
b = _measure(LICH, strengthened=True)[3]
|
||||
assert a == b
|
||||
+78
-1
@@ -22,7 +22,13 @@ from pf1e_simulator.models import (
|
||||
DamageComponent,
|
||||
Saves,
|
||||
)
|
||||
from pf1e_simulator.runner import EncounterSpec, Side, build_states, run_encounter
|
||||
from pf1e_simulator.runner import (
|
||||
EncounterReport,
|
||||
EncounterSpec,
|
||||
Side,
|
||||
build_states,
|
||||
run_encounter,
|
||||
)
|
||||
|
||||
|
||||
def make_combatant(
|
||||
@@ -391,3 +397,74 @@ def test_run_encounter_reports_healing_done() -> None:
|
||||
report = run_encounter(spec, runs=5, seed=11, policy=_self_healing_policy)
|
||||
assert report.attrition["healer"].healing_done > 0.0
|
||||
assert report.attrition["healer"].damage_taken > 0.0
|
||||
|
||||
|
||||
def _duel_spec() -> EncounterSpec:
|
||||
return EncounterSpec(
|
||||
map=make_map(),
|
||||
sides=(
|
||||
Side(name="players", combatants=(make_combatant("p1", hp=20),), zone="A"),
|
||||
Side(name="monsters", combatants=(make_combatant("m1", hp=20),), zone="B"),
|
||||
),
|
||||
round_cap=20,
|
||||
)
|
||||
|
||||
|
||||
def test_run_encounter_default_unchanged_without_collect_runs() -> None:
|
||||
"""Without collect_runs the call still returns the bare EncounterReport."""
|
||||
result = run_encounter(_duel_spec(), runs=5, seed=3)
|
||||
assert isinstance(result, EncounterReport)
|
||||
|
||||
|
||||
def test_run_encounter_collect_runs_returns_report_and_summaries() -> None:
|
||||
"""collect_runs=True returns (report, per-run summaries) with consistent aggregates."""
|
||||
spec = _duel_spec()
|
||||
report, outcomes = run_encounter(spec, runs=10, seed=3, collect_runs=True)
|
||||
assert isinstance(report, EncounterReport)
|
||||
assert len(outcomes) == 10
|
||||
wins = sum(1 for o in outcomes if o.winner == "players")
|
||||
draws = sum(1 for o in outcomes if o.winner is None)
|
||||
assert wins == report.wins["players"]
|
||||
assert draws == report.draws
|
||||
assert sum(o.rounds for o in outcomes) / 10 == report.avg_rounds
|
||||
for o in outcomes:
|
||||
assert {c.id for c in o.combatants} == {"p1", "m1"}
|
||||
by_id = {c.id: c for c in o.combatants}
|
||||
assert by_id["p1"].side == "players"
|
||||
assert by_id["m1"].side == "monsters"
|
||||
assert by_id["p1"].hp_max == 20
|
||||
# dead is the engine death condition: hp <= -CON (con 12 here)
|
||||
for c in o.combatants:
|
||||
assert c.dead == (c.hp <= -12)
|
||||
|
||||
|
||||
def test_run_encounter_collect_runs_deterministic() -> None:
|
||||
"""Same seed -> identical per-run summaries."""
|
||||
spec = _duel_spec()
|
||||
_, a = run_encounter(spec, runs=8, seed=9, collect_runs=True)
|
||||
_, b = run_encounter(spec, runs=8, seed=9, collect_runs=True)
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_run_encounter_collect_runs_dead_matches_winner() -> None:
|
||||
"""A side that never wins still ends runs with its member dead in the summaries."""
|
||||
spec = EncounterSpec(
|
||||
map=make_map(),
|
||||
sides=(
|
||||
Side(name="players", combatants=(make_combatant("weak", hp=4),), zone="A"),
|
||||
Side(
|
||||
name="monsters",
|
||||
combatants=(make_combatant("strong", hp=40, attack_bonus=10, damage="2d6"),) * 2,
|
||||
zone="B",
|
||||
),
|
||||
),
|
||||
round_cap=20,
|
||||
)
|
||||
report, outcomes = run_encounter(spec, runs=5, seed=5, collect_runs=True)
|
||||
assert report.wins["monsters"] == 5
|
||||
for o in outcomes:
|
||||
weak = next(c for c in o.combatants if c.id == "weak")
|
||||
# PF1e: the battle ends when weak is INACTIVE (hp < 0), not necessarily
|
||||
# dead - dead must mirror the engine condition hp <= -CON, not the loss.
|
||||
assert weak.hp < 0
|
||||
assert weak.dead == (weak.hp <= -12)
|
||||
|
||||
Reference in New Issue
Block a user