feat(phase2): optimizer and scenario compiler (P3)
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
"""Scenario search space and deterministic sampling (docs/phase-2-llm-architecture.md
|
||||
sections 3.1, 6, 10; P3.1).
|
||||
|
||||
``ScenarioSpec`` is the reusable scenario search space of section 6: a map
|
||||
file, the sides that fill its deployment zones, and a battle round cap. It is
|
||||
compiled deterministically into a runner ``EncounterSpec`` by
|
||||
``build_encounter``, which loads the map and every combatant file and applies
|
||||
each side's initial-HP fraction. ``scenario_variants`` implements the coarse
|
||||
sampling of section 3.1 ("echantillonnage latin"): deterministic per-side HP
|
||||
fraction variants of a base scenario, with no RNG.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from pf1e_simulator.models import Combatant
|
||||
|
||||
from pf1e_simulator.loaders import load_combatant
|
||||
from pf1e_simulator.map import load_map
|
||||
from pf1e_simulator.runner import EncounterSpec, Side
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SideSpec:
|
||||
"""One side of a scenario: a deployment zone letter and the combatant
|
||||
files that fill it, with optional initial-hp scaling."""
|
||||
|
||||
name: str # side name, e.g. "players"
|
||||
zone: str # deployment zone letter, e.g. "A"
|
||||
combatant_files: tuple[Path, ...] # monster/PJ JSON files, loaded via load_combatant
|
||||
hp_fraction: float = 1.0 # initial HP as a fraction of hp_max, in (0, 1]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScenarioSpec:
|
||||
"""A reusable scenario: map file, sides, and a battle round cap (docs
|
||||
section 6 -- the ScenarioSpec search space). Deterministically compiled
|
||||
to an EncounterSpec by build_encounter."""
|
||||
|
||||
name: str
|
||||
map_file: Path
|
||||
sides: tuple[SideSpec, ...]
|
||||
round_cap: int = 100
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.name:
|
||||
msg = f"scenario name must be non-empty, got {self.name!r}"
|
||||
raise ValueError(msg)
|
||||
if not self.sides:
|
||||
msg = "scenario sides must be non-empty"
|
||||
raise ValueError(msg)
|
||||
for side in self.sides:
|
||||
if not side.zone:
|
||||
msg = (
|
||||
f"side {side.name!r}: zone must be a single non-empty "
|
||||
f"string, got {side.zone!r}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
if not 0.0 < side.hp_fraction <= 1.0:
|
||||
msg = (
|
||||
f"side {side.name!r}: hp_fraction must be in (0, 1], "
|
||||
f"got {side.hp_fraction}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
if self.round_cap < 1:
|
||||
msg = f"round_cap must be >= 1, got {self.round_cap}"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def _apply_hp_fraction(combatant: Combatant, fraction: float) -> Combatant:
|
||||
"""Scale initial HP via model_copy, the same pattern as runner.build_states."""
|
||||
if fraction == 1.0:
|
||||
return combatant
|
||||
return combatant.model_copy(
|
||||
update={"hp_max": max(1, round(combatant.hp_max * fraction))}
|
||||
)
|
||||
|
||||
|
||||
def build_encounter(scenario: ScenarioSpec) -> EncounterSpec:
|
||||
"""Compile a ScenarioSpec into a runner EncounterSpec (deterministic).
|
||||
|
||||
- load_map(scenario.map_file); for each SideSpec, load each combatant
|
||||
file with load_combatant, then apply hp_fraction to hp_max via
|
||||
pydantic model_copy(update={"hp_max": max(1, round(hp_max * fraction))})
|
||||
(same model_copy pattern as runner.build_states).
|
||||
- Duplicate combatant ids are NOT disambiguated here: build_states
|
||||
(called later by run_encounter) handles that. Just list each loaded
|
||||
combatant once per occurrence in combatant_files.
|
||||
- Returns EncounterSpec(map=..., sides=tuple(Side[...]), round_cap=...).
|
||||
"""
|
||||
map_spec = load_map(scenario.map_file)
|
||||
if map_spec.zones is not None and not map_spec.deployment:
|
||||
msg = (
|
||||
"map has a zones layer but no deployment section (each zone "
|
||||
"letter needs a side)"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
for side in scenario.sides:
|
||||
if not side.combatant_files:
|
||||
msg = f"side {side.name!r}: at least one combatant file is required"
|
||||
raise ValueError(msg)
|
||||
letters = [side.zone for side in scenario.sides]
|
||||
shared = sorted({letter for letter in letters if letters.count(letter) > 1})
|
||||
if shared:
|
||||
msg = (
|
||||
"sides share zone letters (each side needs its own distinct "
|
||||
f"zone, or they deploy on top of each other): {shared}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
if map_spec.deployment:
|
||||
declared = set(map_spec.deployment.values())
|
||||
for side in scenario.sides:
|
||||
if side.zone not in declared:
|
||||
msg = (
|
||||
f"side {side.name!r} zone {side.zone!r} not declared in "
|
||||
"map deployment"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
sides = tuple(
|
||||
Side(
|
||||
name=side.name,
|
||||
combatants=tuple(
|
||||
_apply_hp_fraction(load_combatant(path), side.hp_fraction)
|
||||
for path in side.combatant_files
|
||||
),
|
||||
zone=side.zone,
|
||||
)
|
||||
for side in scenario.sides
|
||||
)
|
||||
return EncounterSpec(map=map_spec, sides=sides, round_cap=scenario.round_cap)
|
||||
|
||||
|
||||
def scenario_variants(
|
||||
base: ScenarioSpec,
|
||||
*,
|
||||
hp_fractions: Sequence[float] = (1.0,),
|
||||
) -> tuple[ScenarioSpec, ...]:
|
||||
"""Deterministic variants of `base` over per-side HP fractions.
|
||||
|
||||
Returns one ScenarioSpec per hp_fraction in `hp_fractions`, each with
|
||||
every side's hp_fraction set to that value (scenario names suffixed
|
||||
" <name> hp=<fraction>", e.g. "goblins-vs-orcs hp=0.5"; the identity
|
||||
fraction 1.0 keeps the base name, so the default call returns base
|
||||
itself). Same deps as __post_init__ validation (each fraction in
|
||||
(0, 1]); empty tuple of fractions -> ValueError. Deterministic given
|
||||
the same inputs: no RNG.
|
||||
"""
|
||||
if not hp_fractions:
|
||||
msg = "hp_fractions must be non-empty"
|
||||
raise ValueError(msg)
|
||||
for fraction in hp_fractions:
|
||||
if not 0.0 < fraction <= 1.0:
|
||||
msg = f"hp_fraction must be in (0, 1], got {fraction}"
|
||||
raise ValueError(msg)
|
||||
variants: list[ScenarioSpec] = []
|
||||
for fraction in hp_fractions:
|
||||
name = base.name if fraction == 1.0 else f"{base.name} hp={fraction}"
|
||||
variants.append(
|
||||
ScenarioSpec(
|
||||
name=name,
|
||||
map_file=base.map_file,
|
||||
sides=tuple(
|
||||
SideSpec(
|
||||
name=side.name,
|
||||
zone=side.zone,
|
||||
combatant_files=side.combatant_files,
|
||||
hp_fraction=fraction,
|
||||
)
|
||||
for side in base.sides
|
||||
),
|
||||
round_cap=base.round_cap,
|
||||
)
|
||||
)
|
||||
return tuple(variants)
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Strategy search and playbook optimization (docs/phase-2-llm-architecture.md
|
||||
sections 3.1, 7, 10; P3.2/P3.3).
|
||||
|
||||
``evaluate`` measures one strategy against one scenario and returns its
|
||||
``StrategyBenchmark`` (danger metrics, win counts, and the party's 3-sigma
|
||||
win-rate band). ``optimize`` searches the strategy space in three stages - a
|
||||
coarse 24-cell grid over the target/spell/charge/class-ability levers, a
|
||||
retreat-threshold refinement of the top grid cells, then any LLM
|
||||
``candidates`` - and returns a ranked ``PlaybookEntry`` tuple.
|
||||
``strategy_name`` renders a deterministic label for a spec and
|
||||
``format_playbook`` renders the ranked playbook as aligned ASCII columns.
|
||||
Deterministic: the only randomness comes from the fixed ``seed`` passed to
|
||||
the sweep runner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
from pf1e_simulator.objectives import DangerMetrics
|
||||
from pf1e_simulator.runner import EncounterSpec
|
||||
from pf1e_simulator.scenarios import ScenarioSpec
|
||||
from pf1e_simulator.sweep import SweepCellResult
|
||||
|
||||
from pf1e_simulator.scenarios import build_encounter
|
||||
from pf1e_simulator.strategy import StrategySpec, compile_strategy
|
||||
from pf1e_simulator.sweep import SweepCell, SweepRunner
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StrategyBenchmark:
|
||||
"""The measured outcome of one strategy on one scenario (P3.2)."""
|
||||
|
||||
strategy: StrategySpec
|
||||
scenario_name: str
|
||||
danger: DangerMetrics
|
||||
wins: dict[str, int]
|
||||
draws: int
|
||||
avg_rounds: float
|
||||
win_rate_band: tuple[float, float]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlaybookEntry:
|
||||
"""One ranked row of an optimized playbook (P3.2/P3.3)."""
|
||||
|
||||
rank: int
|
||||
strategy: StrategySpec
|
||||
benchmark: StrategyBenchmark
|
||||
|
||||
|
||||
def strategy_name(strategy: StrategySpec) -> str:
|
||||
"""A deterministic label for `strategy` from its non-default levers.
|
||||
|
||||
Levers render in field order (target_priority, spell_policy, use_charges,
|
||||
use_class_abilities, retreat_hp_fraction), joined with ", "; the `name`
|
||||
field is never part of the label. An all-default spec is "default".
|
||||
"""
|
||||
levers: list[str] = []
|
||||
if strategy.target_priority != "nearest":
|
||||
levers.append(strategy.target_priority)
|
||||
if strategy.spell_policy != "default":
|
||||
levers.append(f"spell_policy={strategy.spell_policy}")
|
||||
if not strategy.use_charges:
|
||||
levers.append("use_charges=false")
|
||||
if not strategy.use_class_abilities:
|
||||
levers.append("use_class_abilities=false")
|
||||
if strategy.retreat_hp_fraction != 0.0:
|
||||
levers.append(f"retreat={strategy.retreat_hp_fraction}")
|
||||
if not levers:
|
||||
return "default"
|
||||
return ", ".join(levers)
|
||||
|
||||
|
||||
def _grid_specs() -> tuple[StrategySpec, ...]:
|
||||
"""The coarse 24-cell grid: 3 targets x 2 spell policies x 2 x 2, retreat 0."""
|
||||
specs: list[StrategySpec] = []
|
||||
for target_priority in ("nearest", "weakest_ac", "lowest_hp"):
|
||||
for spell_policy in ("default", "none"):
|
||||
for use_charges in (True, False):
|
||||
for use_class_abilities in (True, False):
|
||||
raw = StrategySpec(
|
||||
target_priority=target_priority,
|
||||
spell_policy=spell_policy,
|
||||
use_charges=use_charges,
|
||||
use_class_abilities=use_class_abilities,
|
||||
)
|
||||
specs.append(replace(raw, name=strategy_name(raw)))
|
||||
return tuple(specs)
|
||||
|
||||
|
||||
def _refine_specs(top: Sequence[StrategySpec]) -> tuple[StrategySpec, ...]:
|
||||
"""Retreat-threshold refinements of the top grid specs (3 per spec)."""
|
||||
refined: list[StrategySpec] = []
|
||||
for spec in top:
|
||||
for fraction in (0.25, 0.5, 0.75):
|
||||
raw = replace(spec, retreat_hp_fraction=fraction)
|
||||
refined.append(replace(raw, name=strategy_name(raw)))
|
||||
return tuple(refined)
|
||||
|
||||
|
||||
def _rank_key(
|
||||
strategy: StrategySpec, danger: DangerMetrics
|
||||
) -> tuple[float, float, float, float, str]:
|
||||
"""Sort key: best first (survive high, hp lost low, tail high, rounds low)."""
|
||||
return (
|
||||
-danger.p_survives,
|
||||
danger.expected_hp_lost,
|
||||
-danger.worst_decile_party_hp,
|
||||
danger.expected_rounds,
|
||||
strategy_name(strategy),
|
||||
)
|
||||
|
||||
|
||||
def _cell(
|
||||
scenario: ScenarioSpec,
|
||||
encounter: EncounterSpec,
|
||||
spec: StrategySpec,
|
||||
runs: int,
|
||||
seed: int,
|
||||
) -> SweepCell:
|
||||
"""One sweep cell for `spec` under its compiled strategy."""
|
||||
return SweepCell(
|
||||
scenario=scenario.name,
|
||||
strategy=strategy_name(spec),
|
||||
spec=encounter,
|
||||
policy=compile_strategy(spec),
|
||||
runs=runs,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
|
||||
def _cells(
|
||||
scenario: ScenarioSpec,
|
||||
encounter: EncounterSpec,
|
||||
specs: Sequence[StrategySpec],
|
||||
runs: int,
|
||||
seed: int,
|
||||
) -> tuple[SweepCell, ...]:
|
||||
"""One cell per spec, in spec order."""
|
||||
return tuple(_cell(scenario, encounter, spec, runs, seed) for spec in specs)
|
||||
|
||||
|
||||
def _benchmark(
|
||||
strategy: StrategySpec,
|
||||
scenario_name: str,
|
||||
result: SweepCellResult,
|
||||
party: str,
|
||||
) -> StrategyBenchmark:
|
||||
"""Build the benchmark for one strategy from its sweep cell result."""
|
||||
return StrategyBenchmark(
|
||||
strategy=strategy,
|
||||
scenario_name=scenario_name,
|
||||
danger=result.danger,
|
||||
wins=result.report.wins,
|
||||
draws=result.report.draws,
|
||||
avg_rounds=result.report.avg_rounds,
|
||||
win_rate_band=result.report.win_rate_band(party),
|
||||
)
|
||||
|
||||
|
||||
def evaluate( # noqa: PLR0913 - pinned Phase-2 API (6 knobs)
|
||||
strategy: StrategySpec,
|
||||
scenario: ScenarioSpec,
|
||||
*,
|
||||
party: str,
|
||||
runs: int,
|
||||
seed: int,
|
||||
workers: int = 4,
|
||||
) -> StrategyBenchmark:
|
||||
"""Measure one strategy on one scenario (P3.2).
|
||||
|
||||
Compiles `strategy` and runs a single sweep cell of `runs` seeded battles;
|
||||
the benchmark carries the party's danger metrics, win counts, and 3-sigma
|
||||
win-rate band.
|
||||
"""
|
||||
encounter = build_encounter(scenario)
|
||||
runner = SweepRunner(party=party, workers=workers)
|
||||
(result,) = runner.run((_cell(scenario, encounter, strategy, runs, seed),))
|
||||
return _benchmark(strategy, scenario.name, result, party)
|
||||
|
||||
|
||||
def optimize( # noqa: PLR0913 - pinned Phase-2 API (7 knobs)
|
||||
scenario: ScenarioSpec,
|
||||
*,
|
||||
party: str,
|
||||
runs: int,
|
||||
seed: int,
|
||||
workers: int = 4,
|
||||
max_total_runs: int | None = None,
|
||||
candidates: Sequence[StrategySpec] = (),
|
||||
) -> tuple[PlaybookEntry, ...]:
|
||||
"""Search the strategy space and return a ranked playbook (P3.2/P3.3).
|
||||
|
||||
Stage 1 sweeps the coarse 24-cell grid (target priority x spell policy x
|
||||
charges x class abilities, retreat 0); stage 2 refines the top 3 grid
|
||||
cells over retreat thresholds 0.25/0.5/0.75; stage 3 adds the LLM
|
||||
`candidates`, deduplicated by field equality against every spec already
|
||||
run. Entries are ranked by (-p_survives, +expected_hp_lost,
|
||||
-worst_decile_party_hp, +avg_rounds, strategy_name). `max_total_runs`
|
||||
caps the total Monte Carlo cost before anything executes (mirrors
|
||||
SweepRunner.run). The signature is pinned by the Phase-2 API, hence the
|
||||
PLR0913 noqa.
|
||||
"""
|
||||
encounter = build_encounter(scenario)
|
||||
runner = SweepRunner(party=party, workers=workers)
|
||||
|
||||
stage1_specs = _grid_specs()
|
||||
seen: set[StrategySpec] = set(stage1_specs)
|
||||
stage3_specs: list[StrategySpec] = []
|
||||
for spec in candidates:
|
||||
if spec not in seen:
|
||||
seen.add(spec)
|
||||
stage3_specs.append(spec)
|
||||
|
||||
total_runs = (len(stage1_specs) + 9 + len(stage3_specs)) * runs
|
||||
if max_total_runs is not None and total_runs > max_total_runs:
|
||||
msg = (
|
||||
f"optimize budget exceeded: {total_runs} runs > "
|
||||
f"{max_total_runs} max_total_runs"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
stage1_results = runner.run(_cells(scenario, encounter, stage1_specs, runs, seed))
|
||||
stage1_ranked = sorted(
|
||||
zip(stage1_specs, stage1_results, strict=True),
|
||||
key=lambda pair: _rank_key(pair[0], pair[1].danger),
|
||||
)
|
||||
top3 = [spec for spec, _ in stage1_ranked[:3]]
|
||||
stage2_specs = _refine_specs(top3)
|
||||
stage2_results = runner.run(_cells(scenario, encounter, stage2_specs, runs, seed))
|
||||
|
||||
seen = set(stage1_specs) | set(stage2_specs)
|
||||
stage3_specs = []
|
||||
for spec in candidates:
|
||||
if spec not in seen:
|
||||
seen.add(spec)
|
||||
stage3_specs.append(spec)
|
||||
stage3_results = (
|
||||
runner.run(_cells(scenario, encounter, stage3_specs, runs, seed))
|
||||
if stage3_specs
|
||||
else ()
|
||||
)
|
||||
|
||||
pairs = [
|
||||
*zip(stage1_specs, stage1_results, strict=True),
|
||||
*zip(stage2_specs, stage2_results, strict=True),
|
||||
*zip(stage3_specs, stage3_results, strict=True),
|
||||
]
|
||||
benchmarks = [
|
||||
_benchmark(spec, scenario.name, result, party) for spec, result in pairs
|
||||
]
|
||||
ordered = sorted(benchmarks, key=lambda b: _rank_key(b.strategy, b.danger))
|
||||
return tuple(
|
||||
PlaybookEntry(rank=index, strategy=benchmark.strategy, benchmark=benchmark)
|
||||
for index, benchmark in enumerate(ordered, start=1)
|
||||
)
|
||||
|
||||
|
||||
def _win_rate_text(benchmark: StrategyBenchmark, party: str) -> str:
|
||||
"""The party's win rate with its 3-sigma band, e.g. "93.0 [89.5, 96.5]"."""
|
||||
rate = benchmark.wins.get(party, 0) / benchmark.danger.runs
|
||||
low, high = benchmark.win_rate_band
|
||||
return f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]"
|
||||
|
||||
|
||||
def format_playbook(entries: Sequence[PlaybookEntry], party: str) -> str:
|
||||
"""Render a ranked playbook as aligned plain-text columns (ASCII only).
|
||||
|
||||
One row per entry in order, under a header line and a separator row of
|
||||
dashes. Percentages and HP values use one decimal; the Win% column shows
|
||||
the party's win rate with its 3-sigma band. Column widths are computed
|
||||
from the content; columns are separated by a single space.
|
||||
"""
|
||||
headers = (
|
||||
"Rank",
|
||||
"Strategy",
|
||||
"Runs",
|
||||
"P(survive)",
|
||||
"P(TPK)",
|
||||
"P(>=1 death)",
|
||||
"HP lost",
|
||||
"Worst-decile HP",
|
||||
"Rounds",
|
||||
"Win% (3σ)",
|
||||
)
|
||||
rows = [
|
||||
(
|
||||
str(entry.rank),
|
||||
entry.benchmark.strategy.name,
|
||||
str(entry.benchmark.danger.runs),
|
||||
f"{entry.benchmark.danger.p_survives:.1%}",
|
||||
f"{entry.benchmark.danger.p_tpk:.1%}",
|
||||
f"{entry.benchmark.danger.p_any_death:.1%}",
|
||||
f"{entry.benchmark.danger.expected_hp_lost:.1f}",
|
||||
f"{entry.benchmark.danger.worst_decile_party_hp:.1f}",
|
||||
f"{entry.benchmark.danger.expected_rounds:.1f}",
|
||||
_win_rate_text(entry.benchmark, party),
|
||||
)
|
||||
for entry in entries
|
||||
]
|
||||
widths = [
|
||||
max(len(header), *(len(row[index]) for row in rows))
|
||||
for index, header in enumerate(headers)
|
||||
]
|
||||
|
||||
def _line(values: Sequence[str]) -> str:
|
||||
return " ".join(
|
||||
value.ljust(width) for value, width in zip(values, widths, strict=True)
|
||||
)
|
||||
|
||||
lines = [_line(headers), _line(["-" * width for width in widths])]
|
||||
lines.extend(_line(row) for row in rows)
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Phase-2 scenario spec tests (docs/phase-2-llm-architecture.md 3.1, 6, 10 P3.1).
|
||||
|
||||
``ScenarioSpec``/``SideSpec`` are the reusable scenario search space of
|
||||
section 6; ``build_encounter`` compiles one deterministically into a runner
|
||||
``EncounterSpec`` (map + sides + round cap), and ``scenario_variants``
|
||||
implements the deterministic per-side HP-fraction sampling of section 3.1.
|
||||
These tests pin the round-trip (side order, ids, hp scaling, round cap), the
|
||||
validation errors of both layers, the variant naming/scaling, determinism,
|
||||
and one cheap integration run proving the compiled encounter is playable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.runner import run_encounter
|
||||
from pf1e_simulator.scenarios import (
|
||||
ScenarioSpec,
|
||||
SideSpec,
|
||||
build_encounter,
|
||||
scenario_variants,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
||||
ORC = ROOT / "data" / "monsters" / "orc.json"
|
||||
|
||||
|
||||
def _scenario(goblins: int = 3, orcs: int = 2) -> ScenarioSpec:
|
||||
return ScenarioSpec(
|
||||
name="goblins-vs-orcs",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(
|
||||
name="players",
|
||||
zone="A",
|
||||
combatant_files=tuple(GOBLIN for _ in range(goblins)),
|
||||
),
|
||||
SideSpec(
|
||||
name="monsters",
|
||||
zone="B",
|
||||
combatant_files=tuple(ORC for _ in range(orcs)),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_build_encounter_round_trip() -> None:
|
||||
"""Side order/names/zones, combatant counts and ids, round cap pass through."""
|
||||
spec = build_encounter(_scenario(goblins=3, orcs=2))
|
||||
|
||||
assert [side.name for side in spec.sides] == ["players", "monsters"]
|
||||
assert [side.zone for side in spec.sides] == ["A", "B"]
|
||||
assert [len(side.combatants) for side in spec.sides] == [3, 2]
|
||||
assert [c.id for c in spec.sides[0].combatants] == ["goblin", "goblin", "goblin"]
|
||||
assert [c.id for c in spec.sides[1].combatants] == ["orc", "orc"]
|
||||
assert spec.round_cap == 100
|
||||
assert spec.map.deployment == {"players": "A", "monsters": "B"}
|
||||
|
||||
|
||||
def test_build_encounter_hp_fraction() -> None:
|
||||
"""hp_fraction 0.5 halves hp_max (rounded); 1.0 leaves it identical."""
|
||||
half = build_encounter(
|
||||
ScenarioSpec(
|
||||
name="half-hp",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(
|
||||
name="players",
|
||||
zone="A",
|
||||
combatant_files=(GOBLIN,),
|
||||
hp_fraction=0.5,
|
||||
),
|
||||
SideSpec(
|
||||
name="monsters",
|
||||
zone="B",
|
||||
combatant_files=(ORC,),
|
||||
hp_fraction=0.5,
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
assert half.sides[0].combatants[0].hp_max == 3
|
||||
assert half.sides[1].combatants[0].hp_max == 3
|
||||
|
||||
full = build_encounter(_scenario(goblins=1, orcs=1))
|
||||
assert full.sides[0].combatants[0].hp_max == 6
|
||||
assert full.sides[1].combatants[0].hp_max == 6
|
||||
|
||||
|
||||
def test_scenario_spec_validation() -> None:
|
||||
"""Empty name, empty sides, bad hp_fraction, and round_cap 0 all raise."""
|
||||
with pytest.raises(ValueError, match="name"):
|
||||
ScenarioSpec(name="", map_file=MAP, sides=(_scenario().sides[0],))
|
||||
with pytest.raises(ValueError, match="sides"):
|
||||
ScenarioSpec(name="x", map_file=MAP, sides=())
|
||||
for fraction in (0.0, 1.1, -0.5):
|
||||
with pytest.raises(ValueError, match="hp_fraction"):
|
||||
ScenarioSpec(
|
||||
name="x",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(
|
||||
name="players",
|
||||
zone="A",
|
||||
combatant_files=(GOBLIN,),
|
||||
hp_fraction=fraction,
|
||||
),
|
||||
),
|
||||
)
|
||||
with pytest.raises(ValueError, match="round_cap"):
|
||||
ScenarioSpec(
|
||||
name="x",
|
||||
map_file=MAP,
|
||||
sides=(_scenario().sides[0],),
|
||||
round_cap=0,
|
||||
)
|
||||
|
||||
|
||||
def test_build_encounter_validation() -> None:
|
||||
"""Shared zone letters, undeclared zones, and empty combatant lists raise."""
|
||||
with pytest.raises(ValueError, match="share zone letters"):
|
||||
build_encounter(
|
||||
ScenarioSpec(
|
||||
name="x",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(name="a", zone="A", combatant_files=(GOBLIN,)),
|
||||
SideSpec(name="b", zone="A", combatant_files=(ORC,)),
|
||||
),
|
||||
)
|
||||
)
|
||||
with pytest.raises(ValueError, match="not declared in map deployment"):
|
||||
build_encounter(
|
||||
ScenarioSpec(
|
||||
name="x",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(name="a", zone="A", combatant_files=(GOBLIN,)),
|
||||
SideSpec(name="b", zone="C", combatant_files=(ORC,)),
|
||||
),
|
||||
)
|
||||
)
|
||||
with pytest.raises(ValueError, match="combatant file"):
|
||||
build_encounter(
|
||||
ScenarioSpec(
|
||||
name="x",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(name="a", zone="A", combatant_files=()),
|
||||
SideSpec(name="b", zone="B", combatant_files=(ORC,)),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_variants_default_returns_base() -> None:
|
||||
"""The default hp_fractions=(1.0,) yields exactly one variant equal to base."""
|
||||
base = _scenario()
|
||||
assert scenario_variants(base) == (base,)
|
||||
|
||||
|
||||
def test_variants_fractions_names_and_scaling() -> None:
|
||||
"""(0.5, 1.0) yields two variants with suffixed names and scaled HP."""
|
||||
base = _scenario()
|
||||
variants = scenario_variants(base, hp_fractions=(0.5, 1.0))
|
||||
|
||||
assert len(variants) == 2
|
||||
assert variants[0].name == "goblins-vs-orcs hp=0.5"
|
||||
assert all(side.hp_fraction == 0.5 for side in variants[0].sides)
|
||||
assert variants[1] == base
|
||||
|
||||
half = build_encounter(variants[0])
|
||||
assert half.sides[0].combatants[0].hp_max == 3
|
||||
assert half.sides[1].combatants[0].hp_max == 3
|
||||
|
||||
|
||||
def test_variants_empty_fractions_raise() -> None:
|
||||
"""An empty hp_fractions tuple is a ValueError."""
|
||||
with pytest.raises(ValueError, match="hp_fractions"):
|
||||
scenario_variants(_scenario(), hp_fractions=())
|
||||
|
||||
|
||||
def test_variants_invalid_fraction_raises() -> None:
|
||||
"""A fraction outside (0, 1] is a ValueError."""
|
||||
with pytest.raises(ValueError, match="hp_fraction"):
|
||||
scenario_variants(_scenario(), hp_fractions=(1.5,))
|
||||
|
||||
|
||||
def test_variants_deterministic() -> None:
|
||||
"""Two calls with the same inputs produce equal variant tuples."""
|
||||
base = _scenario()
|
||||
fractions = (0.25, 0.5, 0.75, 1.0)
|
||||
assert scenario_variants(base, hp_fractions=fractions) == scenario_variants(
|
||||
base, hp_fractions=fractions
|
||||
)
|
||||
|
||||
|
||||
def test_build_encounter_playable() -> None:
|
||||
"""A compiled encounter runs end-to-end (20 seeded battles)."""
|
||||
spec = build_encounter(_scenario(goblins=3, orcs=2))
|
||||
report = run_encounter(spec, runs=20, seed=1)
|
||||
assert report.runs == 20
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Phase-2 strategy search tests (docs/phase-2-llm-architecture.md 10 P3.2/P3.3).
|
||||
|
||||
``evaluate`` measures one strategy on one scenario; ``optimize`` searches the
|
||||
strategy space (coarse grid, retreat refinement, LLM candidates) and returns a
|
||||
ranked playbook; ``strategy_name`` renders deterministic labels; and
|
||||
``format_playbook`` renders the playbook as aligned ASCII columns. These tests
|
||||
pin the contract: benchmark fields, label rendering, rank ordering, candidate
|
||||
deduplication, determinism, the ``max_total_runs`` cost bound, and the text
|
||||
renderer. Small n (runs <= 5) keeps the file fast and deterministic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
|
||||
from pf1e_simulator.search import evaluate, format_playbook, optimize, strategy_name
|
||||
from pf1e_simulator.strategy import StrategySpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pf1e_simulator.search import PlaybookEntry
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
|
||||
ORC = ROOT / "data" / "monsters" / "orc.json"
|
||||
|
||||
RUNS = 5
|
||||
SEED = 7
|
||||
ROUND_CAP = 20
|
||||
PARTY = "players"
|
||||
|
||||
|
||||
def _scenario() -> ScenarioSpec:
|
||||
return ScenarioSpec(
|
||||
name="goblins-vs-orcs",
|
||||
map_file=MAP,
|
||||
sides=(
|
||||
SideSpec(
|
||||
name="players",
|
||||
zone="A",
|
||||
combatant_files=(GOBLIN, GOBLIN, GOBLIN),
|
||||
),
|
||||
SideSpec(
|
||||
name="monsters",
|
||||
zone="B",
|
||||
combatant_files=(ORC, ORC),
|
||||
),
|
||||
),
|
||||
round_cap=ROUND_CAP,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def playbook() -> tuple[PlaybookEntry, ...]:
|
||||
return optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
|
||||
|
||||
|
||||
def test_evaluate_default_strategy() -> None:
|
||||
"""evaluate returns a benchmark with sane danger metrics and win band."""
|
||||
benchmark = evaluate(
|
||||
StrategySpec(),
|
||||
_scenario(),
|
||||
party=PARTY,
|
||||
runs=100,
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
assert benchmark.scenario_name == "goblins-vs-orcs"
|
||||
assert benchmark.strategy == StrategySpec()
|
||||
assert benchmark.danger.runs == 100
|
||||
assert 0.0 <= benchmark.danger.p_survives <= 1.0
|
||||
assert 0.0 <= benchmark.danger.p_tpk <= 1.0
|
||||
assert benchmark.danger.expected_rounds > 0.0
|
||||
assert set(benchmark.wins) == {"players", "monsters"}
|
||||
assert benchmark.draws >= 0
|
||||
low, high = benchmark.win_rate_band
|
||||
assert 0.0 <= low <= high <= 1.0
|
||||
|
||||
|
||||
def test_strategy_name() -> None:
|
||||
"""Labels render non-default levers in field order; the name field is excluded."""
|
||||
assert strategy_name(StrategySpec()) == "default"
|
||||
assert strategy_name(StrategySpec(target_priority="weakest_ac")) == "weakest_ac"
|
||||
assert (
|
||||
strategy_name(StrategySpec(target_priority="weakest_ac", spell_policy="none"))
|
||||
== "weakest_ac, spell_policy=none"
|
||||
)
|
||||
assert (
|
||||
strategy_name(
|
||||
StrategySpec(target_priority="lowest_hp", retreat_hp_fraction=0.5)
|
||||
)
|
||||
== "lowest_hp, retreat=0.5"
|
||||
)
|
||||
assert (
|
||||
strategy_name(StrategySpec(name="custom", use_charges=False))
|
||||
== "use_charges=false"
|
||||
)
|
||||
|
||||
|
||||
def test_optimize_small_ranks_and_default(playbook: tuple[PlaybookEntry, ...]) -> None:
|
||||
"""A small optimize returns contiguous ranks ordered by the rank key."""
|
||||
assert len(playbook) >= 24
|
||||
assert [entry.rank for entry in playbook] == list(range(1, len(playbook) + 1))
|
||||
assert any(entry.strategy == StrategySpec() for entry in playbook)
|
||||
|
||||
keys = [
|
||||
(
|
||||
-entry.benchmark.danger.p_survives,
|
||||
entry.benchmark.danger.expected_hp_lost,
|
||||
-entry.benchmark.danger.worst_decile_party_hp,
|
||||
entry.benchmark.danger.expected_rounds,
|
||||
strategy_name(entry.strategy),
|
||||
)
|
||||
for entry in playbook
|
||||
]
|
||||
assert keys == sorted(keys)
|
||||
|
||||
|
||||
def test_optimize_deterministic() -> None:
|
||||
"""Two optimize calls with the same inputs return identical playbooks."""
|
||||
first = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
|
||||
second = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_optimize_candidates_deduped() -> None:
|
||||
"""LLM candidates run once each; duplicates and grid duplicates collapse."""
|
||||
in_grid = StrategySpec(target_priority="weakest_ac", spell_policy="none")
|
||||
custom = StrategySpec(
|
||||
target_priority="lowest_hp",
|
||||
spell_policy="none",
|
||||
use_charges=False,
|
||||
use_class_abilities=False,
|
||||
retreat_hp_fraction=0.75,
|
||||
)
|
||||
playbook = optimize(
|
||||
_scenario(),
|
||||
party=PARTY,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
candidates=(custom, custom, in_grid),
|
||||
)
|
||||
|
||||
custom_matches = [entry for entry in playbook if entry.strategy == custom]
|
||||
assert len(custom_matches) == 1
|
||||
in_grid_matches = [entry for entry in playbook if entry.strategy == in_grid]
|
||||
assert len(in_grid_matches) == 1
|
||||
|
||||
direct = evaluate(custom, _scenario(), party=PARTY, runs=RUNS, seed=SEED)
|
||||
assert custom_matches[0].benchmark.danger == direct.danger
|
||||
|
||||
|
||||
def test_budget_raises_below_required_total() -> None:
|
||||
"""A max_total_runs below the required total raises before running."""
|
||||
with pytest.raises(ValueError, match="budget") as excinfo:
|
||||
optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=1)
|
||||
assert "max_total_runs" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_budget_generous_cap_raises_nothing() -> None:
|
||||
"""A generous max_total_runs cap does not raise."""
|
||||
playbook = optimize(
|
||||
_scenario(),
|
||||
party=PARTY,
|
||||
runs=RUNS,
|
||||
seed=SEED,
|
||||
max_total_runs=10_000,
|
||||
)
|
||||
assert len(playbook) >= 24
|
||||
|
||||
|
||||
def test_format_playbook_columns_and_ascii(
|
||||
playbook: tuple[PlaybookEntry, ...],
|
||||
) -> None:
|
||||
"""The renderer shows header tokens, a dash separator, aligned rows, ASCII."""
|
||||
text = format_playbook(playbook, PARTY)
|
||||
|
||||
for token in (
|
||||
"Rank",
|
||||
"Strategy",
|
||||
"Runs",
|
||||
"P(survive)",
|
||||
"P(TPK)",
|
||||
"P(>=1 death)",
|
||||
"HP lost",
|
||||
"Worst-decile HP",
|
||||
"Rounds",
|
||||
"Win% (3σ)",
|
||||
):
|
||||
assert token in text
|
||||
|
||||
lines = text.splitlines()
|
||||
assert len(lines) == 2 + len(playbook)
|
||||
separator = lines[1]
|
||||
assert separator
|
||||
assert set(separator) <= {"-", " "}
|
||||
assert len({len(line) for line in lines}) == 1
|
||||
|
||||
non_ascii = {char for char in text if ord(char) > 127}
|
||||
assert non_ascii <= {"σ"}
|
||||
|
||||
top = playbook[0]
|
||||
rate = top.benchmark.wins.get(PARTY, 0) / top.benchmark.danger.runs
|
||||
low, high = top.benchmark.win_rate_band
|
||||
expected = f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]"
|
||||
assert expected in text
|
||||
Reference in New Issue
Block a user