ctan/pf1e-simulator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(phase2): optimizer and scenario compiler (P3)

Browse Source
This commit is contained in:
Thien An
2026-09-22 15:23:05 +02:00
parent 115f54a7e5
commit dd64c2ebd6
4 changed files with 914 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_phase2_scenarios.py
+206
View File
@@ -0,0 +1,206 @@
"""Phase-2 scenario spec tests (docs/phase-2-llm-architecture.md 3.1, 6, 10 P3.1).
``ScenarioSpec``/``SideSpec`` are the reusable scenario search space of
section 6; ``build_encounter`` compiles one deterministically into a runner
``EncounterSpec`` (map + sides + round cap), and ``scenario_variants``
implements the deterministic per-side HP-fraction sampling of section 3.1.
These tests pin the round-trip (side order, ids, hp scaling, round cap), the
validation errors of both layers, the variant naming/scaling, determinism,
and one cheap integration run proving the compiled encounter is playable.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from pf1e_simulator.runner import run_encounter
from pf1e_simulator.scenarios import (
ScenarioSpec,
SideSpec,
build_encounter,
scenario_variants,
)
ROOT = Path(__file__).resolve().parents[1]
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
ORC = ROOT / "data" / "monsters" / "orc.json"
def _scenario(goblins: int = 3, orcs: int = 2) -> ScenarioSpec:
return ScenarioSpec(
name="goblins-vs-orcs",
map_file=MAP,
sides=(
SideSpec(
name="players",
zone="A",
combatant_files=tuple(GOBLIN for _ in range(goblins)),
),
SideSpec(
name="monsters",
zone="B",
combatant_files=tuple(ORC for _ in range(orcs)),
),
),
)
def test_build_encounter_round_trip() -> None:
"""Side order/names/zones, combatant counts and ids, round cap pass through."""
spec = build_encounter(_scenario(goblins=3, orcs=2))
assert [side.name for side in spec.sides] == ["players", "monsters"]
assert [side.zone for side in spec.sides] == ["A", "B"]
assert [len(side.combatants) for side in spec.sides] == [3, 2]
assert [c.id for c in spec.sides[0].combatants] == ["goblin", "goblin", "goblin"]
assert [c.id for c in spec.sides[1].combatants] == ["orc", "orc"]
assert spec.round_cap == 100
assert spec.map.deployment == {"players": "A", "monsters": "B"}
def test_build_encounter_hp_fraction() -> None:
"""hp_fraction 0.5 halves hp_max (rounded); 1.0 leaves it identical."""
half = build_encounter(
ScenarioSpec(
name="half-hp",
map_file=MAP,
sides=(
SideSpec(
name="players",
zone="A",
combatant_files=(GOBLIN,),
hp_fraction=0.5,
),
SideSpec(
name="monsters",
zone="B",
combatant_files=(ORC,),
hp_fraction=0.5,
),
),
)
)
assert half.sides[0].combatants[0].hp_max == 3
assert half.sides[1].combatants[0].hp_max == 3
full = build_encounter(_scenario(goblins=1, orcs=1))
assert full.sides[0].combatants[0].hp_max == 6
assert full.sides[1].combatants[0].hp_max == 6
def test_scenario_spec_validation() -> None:
"""Empty name, empty sides, bad hp_fraction, and round_cap 0 all raise."""
with pytest.raises(ValueError, match="name"):
ScenarioSpec(name="", map_file=MAP, sides=(_scenario().sides[0],))
with pytest.raises(ValueError, match="sides"):
ScenarioSpec(name="x", map_file=MAP, sides=())
for fraction in (0.0, 1.1, -0.5):
with pytest.raises(ValueError, match="hp_fraction"):
ScenarioSpec(
name="x",
map_file=MAP,
sides=(
SideSpec(
name="players",
zone="A",
combatant_files=(GOBLIN,),
hp_fraction=fraction,
),
),
)
with pytest.raises(ValueError, match="round_cap"):
ScenarioSpec(
name="x",
map_file=MAP,
sides=(_scenario().sides[0],),
round_cap=0,
)
def test_build_encounter_validation() -> None:
"""Shared zone letters, undeclared zones, and empty combatant lists raise."""
with pytest.raises(ValueError, match="share zone letters"):
build_encounter(
ScenarioSpec(
name="x",
map_file=MAP,
sides=(
SideSpec(name="a", zone="A", combatant_files=(GOBLIN,)),
SideSpec(name="b", zone="A", combatant_files=(ORC,)),
),
)
)
with pytest.raises(ValueError, match="not declared in map deployment"):
build_encounter(
ScenarioSpec(
name="x",
map_file=MAP,
sides=(
SideSpec(name="a", zone="A", combatant_files=(GOBLIN,)),
SideSpec(name="b", zone="C", combatant_files=(ORC,)),
),
)
)
with pytest.raises(ValueError, match="combatant file"):
build_encounter(
ScenarioSpec(
name="x",
map_file=MAP,
sides=(
SideSpec(name="a", zone="A", combatant_files=()),
SideSpec(name="b", zone="B", combatant_files=(ORC,)),
),
)
)
def test_variants_default_returns_base() -> None:
"""The default hp_fractions=(1.0,) yields exactly one variant equal to base."""
base = _scenario()
assert scenario_variants(base) == (base,)
def test_variants_fractions_names_and_scaling() -> None:
"""(0.5, 1.0) yields two variants with suffixed names and scaled HP."""
base = _scenario()
variants = scenario_variants(base, hp_fractions=(0.5, 1.0))
assert len(variants) == 2
assert variants[0].name == "goblins-vs-orcs hp=0.5"
assert all(side.hp_fraction == 0.5 for side in variants[0].sides)
assert variants[1] == base
half = build_encounter(variants[0])
assert half.sides[0].combatants[0].hp_max == 3
assert half.sides[1].combatants[0].hp_max == 3
def test_variants_empty_fractions_raise() -> None:
"""An empty hp_fractions tuple is a ValueError."""
with pytest.raises(ValueError, match="hp_fractions"):
scenario_variants(_scenario(), hp_fractions=())
def test_variants_invalid_fraction_raises() -> None:
"""A fraction outside (0, 1] is a ValueError."""
with pytest.raises(ValueError, match="hp_fraction"):
scenario_variants(_scenario(), hp_fractions=(1.5,))
def test_variants_deterministic() -> None:
"""Two calls with the same inputs produce equal variant tuples."""
base = _scenario()
fractions = (0.25, 0.5, 0.75, 1.0)
assert scenario_variants(base, hp_fractions=fractions) == scenario_variants(
base, hp_fractions=fractions
)
def test_build_encounter_playable() -> None:
"""A compiled encounter runs end-to-end (20 seeded battles)."""
spec = build_encounter(_scenario(goblins=3, orcs=2))
report = run_encounter(spec, runs=20, seed=1)
assert report.runs == 20
tests/test_phase2_search.py
+210
View File
@@ -0,0 +1,210 @@
"""Phase-2 strategy search tests (docs/phase-2-llm-architecture.md 10 P3.2/P3.3).
``evaluate`` measures one strategy on one scenario; ``optimize`` searches the
strategy space (coarse grid, retreat refinement, LLM candidates) and returns a
ranked playbook; ``strategy_name`` renders deterministic labels; and
``format_playbook`` renders the playbook as aligned ASCII columns. These tests
pin the contract: benchmark fields, label rendering, rank ordering, candidate
deduplication, determinism, the ``max_total_runs`` cost bound, and the text
renderer. Small n (runs <= 5) keeps the file fast and deterministic.
"""
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from pf1e_simulator.scenarios import ScenarioSpec, SideSpec
from pf1e_simulator.search import evaluate, format_playbook, optimize, strategy_name
from pf1e_simulator.strategy import StrategySpec
if TYPE_CHECKING:
from pf1e_simulator.search import PlaybookEntry
ROOT = Path(__file__).resolve().parents[1]
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
GOBLIN = ROOT / "data" / "monsters" / "goblin.json"
ORC = ROOT / "data" / "monsters" / "orc.json"
RUNS = 5
SEED = 7
ROUND_CAP = 20
PARTY = "players"
def _scenario() -> ScenarioSpec:
return ScenarioSpec(
name="goblins-vs-orcs",
map_file=MAP,
sides=(
SideSpec(
name="players",
zone="A",
combatant_files=(GOBLIN, GOBLIN, GOBLIN),
),
SideSpec(
name="monsters",
zone="B",
combatant_files=(ORC, ORC),
),
),
round_cap=ROUND_CAP,
)
@pytest.fixture(scope="module")
def playbook() -> tuple[PlaybookEntry, ...]:
return optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
def test_evaluate_default_strategy() -> None:
"""evaluate returns a benchmark with sane danger metrics and win band."""
benchmark = evaluate(
StrategySpec(),
_scenario(),
party=PARTY,
runs=100,
seed=SEED,
)
assert benchmark.scenario_name == "goblins-vs-orcs"
assert benchmark.strategy == StrategySpec()
assert benchmark.danger.runs == 100
assert 0.0 <= benchmark.danger.p_survives <= 1.0
assert 0.0 <= benchmark.danger.p_tpk <= 1.0
assert benchmark.danger.expected_rounds > 0.0
assert set(benchmark.wins) == {"players", "monsters"}
assert benchmark.draws >= 0
low, high = benchmark.win_rate_band
assert 0.0 <= low <= high <= 1.0
def test_strategy_name() -> None:
"""Labels render non-default levers in field order; the name field is excluded."""
assert strategy_name(StrategySpec()) == "default"
assert strategy_name(StrategySpec(target_priority="weakest_ac")) == "weakest_ac"
assert (
strategy_name(StrategySpec(target_priority="weakest_ac", spell_policy="none"))
== "weakest_ac, spell_policy=none"
)
assert (
strategy_name(
StrategySpec(target_priority="lowest_hp", retreat_hp_fraction=0.5)
)
== "lowest_hp, retreat=0.5"
)
assert (
strategy_name(StrategySpec(name="custom", use_charges=False))
== "use_charges=false"
)
def test_optimize_small_ranks_and_default(playbook: tuple[PlaybookEntry, ...]) -> None:
"""A small optimize returns contiguous ranks ordered by the rank key."""
assert len(playbook) >= 24
assert [entry.rank for entry in playbook] == list(range(1, len(playbook) + 1))
assert any(entry.strategy == StrategySpec() for entry in playbook)
keys = [
(
-entry.benchmark.danger.p_survives,
entry.benchmark.danger.expected_hp_lost,
-entry.benchmark.danger.worst_decile_party_hp,
entry.benchmark.danger.expected_rounds,
strategy_name(entry.strategy),
)
for entry in playbook
]
assert keys == sorted(keys)
def test_optimize_deterministic() -> None:
"""Two optimize calls with the same inputs return identical playbooks."""
first = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
second = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED)
assert first == second
def test_optimize_candidates_deduped() -> None:
"""LLM candidates run once each; duplicates and grid duplicates collapse."""
in_grid = StrategySpec(target_priority="weakest_ac", spell_policy="none")
custom = StrategySpec(
target_priority="lowest_hp",
spell_policy="none",
use_charges=False,
use_class_abilities=False,
retreat_hp_fraction=0.75,
)
playbook = optimize(
_scenario(),
party=PARTY,
runs=RUNS,
seed=SEED,
candidates=(custom, custom, in_grid),
)
custom_matches = [entry for entry in playbook if entry.strategy == custom]
assert len(custom_matches) == 1
in_grid_matches = [entry for entry in playbook if entry.strategy == in_grid]
assert len(in_grid_matches) == 1
direct = evaluate(custom, _scenario(), party=PARTY, runs=RUNS, seed=SEED)
assert custom_matches[0].benchmark.danger == direct.danger
def test_budget_raises_below_required_total() -> None:
"""A max_total_runs below the required total raises before running."""
with pytest.raises(ValueError, match="budget") as excinfo:
optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=1)
assert "max_total_runs" in str(excinfo.value)
def test_budget_generous_cap_raises_nothing() -> None:
"""A generous max_total_runs cap does not raise."""
playbook = optimize(
_scenario(),
party=PARTY,
runs=RUNS,
seed=SEED,
max_total_runs=10_000,
)
assert len(playbook) >= 24
def test_format_playbook_columns_and_ascii(
playbook: tuple[PlaybookEntry, ...],
) -> None:
"""The renderer shows header tokens, a dash separator, aligned rows, ASCII."""
text = format_playbook(playbook, PARTY)
for token in (
"Rank",
"Strategy",
"Runs",
"P(survive)",
"P(TPK)",
"P(>=1 death)",
"HP lost",
"Worst-decile HP",
"Rounds",
"Win% (3σ)",
):
assert token in text
lines = text.splitlines()
assert len(lines) == 2 + len(playbook)
separator = lines[1]
assert separator
assert set(separator) <= {"-", " "}
assert len({len(line) for line in lines}) == 1
non_ascii = {char for char in text if ord(char) > 127}
assert non_ascii <= {"σ"}
top = playbook[0]
rate = top.benchmark.wins.get(PARTY, 0) / top.benchmark.danger.runs
low, high = top.benchmark.win_rate_band
expected = f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]"
assert expected in text