From dd64c2ebd66f3c7f7402a6d76a397d23b61b3a10 Mon Sep 17 00:00:00 2001 From: ctan Date: Tue, 22 Sep 2026 15:23:05 +0200 Subject: [PATCH] feat(phase2): optimizer and scenario compiler (P3) --- src/pf1e_simulator/scenarios.py | 180 ++++++++++++++++++ src/pf1e_simulator/search.py | 318 ++++++++++++++++++++++++++++++++ tests/test_phase2_scenarios.py | 206 +++++++++++++++++++++ tests/test_phase2_search.py | 210 +++++++++++++++++++++ 4 files changed, 914 insertions(+) create mode 100644 src/pf1e_simulator/scenarios.py create mode 100644 src/pf1e_simulator/search.py create mode 100644 tests/test_phase2_scenarios.py create mode 100644 tests/test_phase2_search.py diff --git a/src/pf1e_simulator/scenarios.py b/src/pf1e_simulator/scenarios.py new file mode 100644 index 0000000..47b18b7 --- /dev/null +++ b/src/pf1e_simulator/scenarios.py @@ -0,0 +1,180 @@ +"""Scenario search space and deterministic sampling (docs/phase-2-llm-architecture.md +sections 3.1, 6, 10; P3.1). + +``ScenarioSpec`` is the reusable scenario search space of section 6: a map +file, the sides that fill its deployment zones, and a battle round cap. It is +compiled deterministically into a runner ``EncounterSpec`` by +``build_encounter``, which loads the map and every combatant file and applies +each side's initial-HP fraction. ``scenario_variants`` implements the coarse +sampling of section 3.1 ("echantillonnage latin"): deterministic per-side HP +fraction variants of a base scenario, with no RNG. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + + from pf1e_simulator.models import Combatant + +from pf1e_simulator.loaders import load_combatant +from pf1e_simulator.map import load_map +from pf1e_simulator.runner import EncounterSpec, Side + + +@dataclass(frozen=True) +class SideSpec: + """One side of a scenario: a deployment zone letter and the combatant + files that fill it, with optional initial-hp scaling.""" + + name: str # side name, e.g. "players" + zone: str # deployment zone letter, e.g. "A" + combatant_files: tuple[Path, ...] # monster/PJ JSON files, loaded via load_combatant + hp_fraction: float = 1.0 # initial HP as a fraction of hp_max, in (0, 1] + + +@dataclass(frozen=True) +class ScenarioSpec: + """A reusable scenario: map file, sides, and a battle round cap (docs + section 6 -- the ScenarioSpec search space). Deterministically compiled + to an EncounterSpec by build_encounter.""" + + name: str + map_file: Path + sides: tuple[SideSpec, ...] + round_cap: int = 100 + + def __post_init__(self) -> None: + if not self.name: + msg = f"scenario name must be non-empty, got {self.name!r}" + raise ValueError(msg) + if not self.sides: + msg = "scenario sides must be non-empty" + raise ValueError(msg) + for side in self.sides: + if not side.zone: + msg = ( + f"side {side.name!r}: zone must be a single non-empty " + f"string, got {side.zone!r}" + ) + raise ValueError(msg) + if not 0.0 < side.hp_fraction <= 1.0: + msg = ( + f"side {side.name!r}: hp_fraction must be in (0, 1], " + f"got {side.hp_fraction}" + ) + raise ValueError(msg) + if self.round_cap < 1: + msg = f"round_cap must be >= 1, got {self.round_cap}" + raise ValueError(msg) + + +def _apply_hp_fraction(combatant: Combatant, fraction: float) -> Combatant: + """Scale initial HP via model_copy, the same pattern as runner.build_states.""" + if fraction == 1.0: + return combatant + return combatant.model_copy( + update={"hp_max": max(1, round(combatant.hp_max * fraction))} + ) + + +def build_encounter(scenario: ScenarioSpec) -> EncounterSpec: + """Compile a ScenarioSpec into a runner EncounterSpec (deterministic). + + - load_map(scenario.map_file); for each SideSpec, load each combatant + file with load_combatant, then apply hp_fraction to hp_max via + pydantic model_copy(update={"hp_max": max(1, round(hp_max * fraction))}) + (same model_copy pattern as runner.build_states). + - Duplicate combatant ids are NOT disambiguated here: build_states + (called later by run_encounter) handles that. Just list each loaded + combatant once per occurrence in combatant_files. + - Returns EncounterSpec(map=..., sides=tuple(Side[...]), round_cap=...). + """ + map_spec = load_map(scenario.map_file) + if map_spec.zones is not None and not map_spec.deployment: + msg = ( + "map has a zones layer but no deployment section (each zone " + "letter needs a side)" + ) + raise ValueError(msg) + for side in scenario.sides: + if not side.combatant_files: + msg = f"side {side.name!r}: at least one combatant file is required" + raise ValueError(msg) + letters = [side.zone for side in scenario.sides] + shared = sorted({letter for letter in letters if letters.count(letter) > 1}) + if shared: + msg = ( + "sides share zone letters (each side needs its own distinct " + f"zone, or they deploy on top of each other): {shared}" + ) + raise ValueError(msg) + if map_spec.deployment: + declared = set(map_spec.deployment.values()) + for side in scenario.sides: + if side.zone not in declared: + msg = ( + f"side {side.name!r} zone {side.zone!r} not declared in " + "map deployment" + ) + raise ValueError(msg) + sides = tuple( + Side( + name=side.name, + combatants=tuple( + _apply_hp_fraction(load_combatant(path), side.hp_fraction) + for path in side.combatant_files + ), + zone=side.zone, + ) + for side in scenario.sides + ) + return EncounterSpec(map=map_spec, sides=sides, round_cap=scenario.round_cap) + + +def scenario_variants( + base: ScenarioSpec, + *, + hp_fractions: Sequence[float] = (1.0,), +) -> tuple[ScenarioSpec, ...]: + """Deterministic variants of `base` over per-side HP fractions. + + Returns one ScenarioSpec per hp_fraction in `hp_fractions`, each with + every side's hp_fraction set to that value (scenario names suffixed + " hp=", e.g. "goblins-vs-orcs hp=0.5"; the identity + fraction 1.0 keeps the base name, so the default call returns base + itself). Same deps as __post_init__ validation (each fraction in + (0, 1]); empty tuple of fractions -> ValueError. Deterministic given + the same inputs: no RNG. + """ + if not hp_fractions: + msg = "hp_fractions must be non-empty" + raise ValueError(msg) + for fraction in hp_fractions: + if not 0.0 < fraction <= 1.0: + msg = f"hp_fraction must be in (0, 1], got {fraction}" + raise ValueError(msg) + variants: list[ScenarioSpec] = [] + for fraction in hp_fractions: + name = base.name if fraction == 1.0 else f"{base.name} hp={fraction}" + variants.append( + ScenarioSpec( + name=name, + map_file=base.map_file, + sides=tuple( + SideSpec( + name=side.name, + zone=side.zone, + combatant_files=side.combatant_files, + hp_fraction=fraction, + ) + for side in base.sides + ), + round_cap=base.round_cap, + ) + ) + return tuple(variants) diff --git a/src/pf1e_simulator/search.py b/src/pf1e_simulator/search.py new file mode 100644 index 0000000..803bfd3 --- /dev/null +++ b/src/pf1e_simulator/search.py @@ -0,0 +1,318 @@ +"""Strategy search and playbook optimization (docs/phase-2-llm-architecture.md +sections 3.1, 7, 10; P3.2/P3.3). + +``evaluate`` measures one strategy against one scenario and returns its +``StrategyBenchmark`` (danger metrics, win counts, and the party's 3-sigma +win-rate band). ``optimize`` searches the strategy space in three stages - a +coarse 24-cell grid over the target/spell/charge/class-ability levers, a +retreat-threshold refinement of the top grid cells, then any LLM +``candidates`` - and returns a ranked ``PlaybookEntry`` tuple. +``strategy_name`` renders a deterministic label for a spec and +``format_playbook`` renders the ranked playbook as aligned ASCII columns. +Deterministic: the only randomness comes from the fixed ``seed`` passed to +the sweep runner. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + + from pf1e_simulator.objectives import DangerMetrics + from pf1e_simulator.runner import EncounterSpec + from pf1e_simulator.scenarios import ScenarioSpec + from pf1e_simulator.sweep import SweepCellResult + +from pf1e_simulator.scenarios import build_encounter +from pf1e_simulator.strategy import StrategySpec, compile_strategy +from pf1e_simulator.sweep import SweepCell, SweepRunner + + +@dataclass(frozen=True) +class StrategyBenchmark: + """The measured outcome of one strategy on one scenario (P3.2).""" + + strategy: StrategySpec + scenario_name: str + danger: DangerMetrics + wins: dict[str, int] + draws: int + avg_rounds: float + win_rate_band: tuple[float, float] + + +@dataclass(frozen=True) +class PlaybookEntry: + """One ranked row of an optimized playbook (P3.2/P3.3).""" + + rank: int + strategy: StrategySpec + benchmark: StrategyBenchmark + + +def strategy_name(strategy: StrategySpec) -> str: + """A deterministic label for `strategy` from its non-default levers. + + Levers render in field order (target_priority, spell_policy, use_charges, + use_class_abilities, retreat_hp_fraction), joined with ", "; the `name` + field is never part of the label. An all-default spec is "default". + """ + levers: list[str] = [] + if strategy.target_priority != "nearest": + levers.append(strategy.target_priority) + if strategy.spell_policy != "default": + levers.append(f"spell_policy={strategy.spell_policy}") + if not strategy.use_charges: + levers.append("use_charges=false") + if not strategy.use_class_abilities: + levers.append("use_class_abilities=false") + if strategy.retreat_hp_fraction != 0.0: + levers.append(f"retreat={strategy.retreat_hp_fraction}") + if not levers: + return "default" + return ", ".join(levers) + + +def _grid_specs() -> tuple[StrategySpec, ...]: + """The coarse 24-cell grid: 3 targets x 2 spell policies x 2 x 2, retreat 0.""" + specs: list[StrategySpec] = [] + for target_priority in ("nearest", "weakest_ac", "lowest_hp"): + for spell_policy in ("default", "none"): + for use_charges in (True, False): + for use_class_abilities in (True, False): + raw = StrategySpec( + target_priority=target_priority, + spell_policy=spell_policy, + use_charges=use_charges, + use_class_abilities=use_class_abilities, + ) + specs.append(replace(raw, name=strategy_name(raw))) + return tuple(specs) + + +def _refine_specs(top: Sequence[StrategySpec]) -> tuple[StrategySpec, ...]: + """Retreat-threshold refinements of the top grid specs (3 per spec).""" + refined: list[StrategySpec] = [] + for spec in top: + for fraction in (0.25, 0.5, 0.75): + raw = replace(spec, retreat_hp_fraction=fraction) + refined.append(replace(raw, name=strategy_name(raw))) + return tuple(refined) + + +def _rank_key( + strategy: StrategySpec, danger: DangerMetrics +) -> tuple[float, float, float, float, str]: + """Sort key: best first (survive high, hp lost low, tail high, rounds low).""" + return ( + -danger.p_survives, + danger.expected_hp_lost, + -danger.worst_decile_party_hp, + danger.expected_rounds, + strategy_name(strategy), + ) + + +def _cell( + scenario: ScenarioSpec, + encounter: EncounterSpec, + spec: StrategySpec, + runs: int, + seed: int, +) -> SweepCell: + """One sweep cell for `spec` under its compiled strategy.""" + return SweepCell( + scenario=scenario.name, + strategy=strategy_name(spec), + spec=encounter, + policy=compile_strategy(spec), + runs=runs, + seed=seed, + ) + + +def _cells( + scenario: ScenarioSpec, + encounter: EncounterSpec, + specs: Sequence[StrategySpec], + runs: int, + seed: int, +) -> tuple[SweepCell, ...]: + """One cell per spec, in spec order.""" + return tuple(_cell(scenario, encounter, spec, runs, seed) for spec in specs) + + +def _benchmark( + strategy: StrategySpec, + scenario_name: str, + result: SweepCellResult, + party: str, +) -> StrategyBenchmark: + """Build the benchmark for one strategy from its sweep cell result.""" + return StrategyBenchmark( + strategy=strategy, + scenario_name=scenario_name, + danger=result.danger, + wins=result.report.wins, + draws=result.report.draws, + avg_rounds=result.report.avg_rounds, + win_rate_band=result.report.win_rate_band(party), + ) + + +def evaluate( # noqa: PLR0913 - pinned Phase-2 API (6 knobs) + strategy: StrategySpec, + scenario: ScenarioSpec, + *, + party: str, + runs: int, + seed: int, + workers: int = 4, +) -> StrategyBenchmark: + """Measure one strategy on one scenario (P3.2). + + Compiles `strategy` and runs a single sweep cell of `runs` seeded battles; + the benchmark carries the party's danger metrics, win counts, and 3-sigma + win-rate band. + """ + encounter = build_encounter(scenario) + runner = SweepRunner(party=party, workers=workers) + (result,) = runner.run((_cell(scenario, encounter, strategy, runs, seed),)) + return _benchmark(strategy, scenario.name, result, party) + + +def optimize( # noqa: PLR0913 - pinned Phase-2 API (7 knobs) + scenario: ScenarioSpec, + *, + party: str, + runs: int, + seed: int, + workers: int = 4, + max_total_runs: int | None = None, + candidates: Sequence[StrategySpec] = (), +) -> tuple[PlaybookEntry, ...]: + """Search the strategy space and return a ranked playbook (P3.2/P3.3). + + Stage 1 sweeps the coarse 24-cell grid (target priority x spell policy x + charges x class abilities, retreat 0); stage 2 refines the top 3 grid + cells over retreat thresholds 0.25/0.5/0.75; stage 3 adds the LLM + `candidates`, deduplicated by field equality against every spec already + run. Entries are ranked by (-p_survives, +expected_hp_lost, + -worst_decile_party_hp, +avg_rounds, strategy_name). `max_total_runs` + caps the total Monte Carlo cost before anything executes (mirrors + SweepRunner.run). The signature is pinned by the Phase-2 API, hence the + PLR0913 noqa. + """ + encounter = build_encounter(scenario) + runner = SweepRunner(party=party, workers=workers) + + stage1_specs = _grid_specs() + seen: set[StrategySpec] = set(stage1_specs) + stage3_specs: list[StrategySpec] = [] + for spec in candidates: + if spec not in seen: + seen.add(spec) + stage3_specs.append(spec) + + total_runs = (len(stage1_specs) + 9 + len(stage3_specs)) * runs + if max_total_runs is not None and total_runs > max_total_runs: + msg = ( + f"optimize budget exceeded: {total_runs} runs > " + f"{max_total_runs} max_total_runs" + ) + raise ValueError(msg) + + stage1_results = runner.run(_cells(scenario, encounter, stage1_specs, runs, seed)) + stage1_ranked = sorted( + zip(stage1_specs, stage1_results, strict=True), + key=lambda pair: _rank_key(pair[0], pair[1].danger), + ) + top3 = [spec for spec, _ in stage1_ranked[:3]] + stage2_specs = _refine_specs(top3) + stage2_results = runner.run(_cells(scenario, encounter, stage2_specs, runs, seed)) + + seen = set(stage1_specs) | set(stage2_specs) + stage3_specs = [] + for spec in candidates: + if spec not in seen: + seen.add(spec) + stage3_specs.append(spec) + stage3_results = ( + runner.run(_cells(scenario, encounter, stage3_specs, runs, seed)) + if stage3_specs + else () + ) + + pairs = [ + *zip(stage1_specs, stage1_results, strict=True), + *zip(stage2_specs, stage2_results, strict=True), + *zip(stage3_specs, stage3_results, strict=True), + ] + benchmarks = [ + _benchmark(spec, scenario.name, result, party) for spec, result in pairs + ] + ordered = sorted(benchmarks, key=lambda b: _rank_key(b.strategy, b.danger)) + return tuple( + PlaybookEntry(rank=index, strategy=benchmark.strategy, benchmark=benchmark) + for index, benchmark in enumerate(ordered, start=1) + ) + + +def _win_rate_text(benchmark: StrategyBenchmark, party: str) -> str: + """The party's win rate with its 3-sigma band, e.g. "93.0 [89.5, 96.5]".""" + rate = benchmark.wins.get(party, 0) / benchmark.danger.runs + low, high = benchmark.win_rate_band + return f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]" + + +def format_playbook(entries: Sequence[PlaybookEntry], party: str) -> str: + """Render a ranked playbook as aligned plain-text columns (ASCII only). + + One row per entry in order, under a header line and a separator row of + dashes. Percentages and HP values use one decimal; the Win% column shows + the party's win rate with its 3-sigma band. Column widths are computed + from the content; columns are separated by a single space. + """ + headers = ( + "Rank", + "Strategy", + "Runs", + "P(survive)", + "P(TPK)", + "P(>=1 death)", + "HP lost", + "Worst-decile HP", + "Rounds", + "Win% (3σ)", + ) + rows = [ + ( + str(entry.rank), + entry.benchmark.strategy.name, + str(entry.benchmark.danger.runs), + f"{entry.benchmark.danger.p_survives:.1%}", + f"{entry.benchmark.danger.p_tpk:.1%}", + f"{entry.benchmark.danger.p_any_death:.1%}", + f"{entry.benchmark.danger.expected_hp_lost:.1f}", + f"{entry.benchmark.danger.worst_decile_party_hp:.1f}", + f"{entry.benchmark.danger.expected_rounds:.1f}", + _win_rate_text(entry.benchmark, party), + ) + for entry in entries + ] + widths = [ + max(len(header), *(len(row[index]) for row in rows)) + for index, header in enumerate(headers) + ] + + def _line(values: Sequence[str]) -> str: + return " ".join( + value.ljust(width) for value, width in zip(values, widths, strict=True) + ) + + lines = [_line(headers), _line(["-" * width for width in widths])] + lines.extend(_line(row) for row in rows) + return "\n".join(lines) diff --git a/tests/test_phase2_scenarios.py b/tests/test_phase2_scenarios.py new file mode 100644 index 0000000..47643b6 --- /dev/null +++ b/tests/test_phase2_scenarios.py @@ -0,0 +1,206 @@ +"""Phase-2 scenario spec tests (docs/phase-2-llm-architecture.md 3.1, 6, 10 P3.1). + +``ScenarioSpec``/``SideSpec`` are the reusable scenario search space of +section 6; ``build_encounter`` compiles one deterministically into a runner +``EncounterSpec`` (map + sides + round cap), and ``scenario_variants`` +implements the deterministic per-side HP-fraction sampling of section 3.1. +These tests pin the round-trip (side order, ids, hp scaling, round cap), the +validation errors of both layers, the variant naming/scaling, determinism, +and one cheap integration run proving the compiled encounter is playable. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pf1e_simulator.runner import run_encounter +from pf1e_simulator.scenarios import ( + ScenarioSpec, + SideSpec, + build_encounter, + scenario_variants, +) + +ROOT = Path(__file__).resolve().parents[1] +MAP = ROOT / "data" / "maps" / "sample_arena.yaml" +GOBLIN = ROOT / "data" / "monsters" / "goblin.json" +ORC = ROOT / "data" / "monsters" / "orc.json" + + +def _scenario(goblins: int = 3, orcs: int = 2) -> ScenarioSpec: + return ScenarioSpec( + name="goblins-vs-orcs", + map_file=MAP, + sides=( + SideSpec( + name="players", + zone="A", + combatant_files=tuple(GOBLIN for _ in range(goblins)), + ), + SideSpec( + name="monsters", + zone="B", + combatant_files=tuple(ORC for _ in range(orcs)), + ), + ), + ) + + +def test_build_encounter_round_trip() -> None: + """Side order/names/zones, combatant counts and ids, round cap pass through.""" + spec = build_encounter(_scenario(goblins=3, orcs=2)) + + assert [side.name for side in spec.sides] == ["players", "monsters"] + assert [side.zone for side in spec.sides] == ["A", "B"] + assert [len(side.combatants) for side in spec.sides] == [3, 2] + assert [c.id for c in spec.sides[0].combatants] == ["goblin", "goblin", "goblin"] + assert [c.id for c in spec.sides[1].combatants] == ["orc", "orc"] + assert spec.round_cap == 100 + assert spec.map.deployment == {"players": "A", "monsters": "B"} + + +def test_build_encounter_hp_fraction() -> None: + """hp_fraction 0.5 halves hp_max (rounded); 1.0 leaves it identical.""" + half = build_encounter( + ScenarioSpec( + name="half-hp", + map_file=MAP, + sides=( + SideSpec( + name="players", + zone="A", + combatant_files=(GOBLIN,), + hp_fraction=0.5, + ), + SideSpec( + name="monsters", + zone="B", + combatant_files=(ORC,), + hp_fraction=0.5, + ), + ), + ) + ) + assert half.sides[0].combatants[0].hp_max == 3 + assert half.sides[1].combatants[0].hp_max == 3 + + full = build_encounter(_scenario(goblins=1, orcs=1)) + assert full.sides[0].combatants[0].hp_max == 6 + assert full.sides[1].combatants[0].hp_max == 6 + + +def test_scenario_spec_validation() -> None: + """Empty name, empty sides, bad hp_fraction, and round_cap 0 all raise.""" + with pytest.raises(ValueError, match="name"): + ScenarioSpec(name="", map_file=MAP, sides=(_scenario().sides[0],)) + with pytest.raises(ValueError, match="sides"): + ScenarioSpec(name="x", map_file=MAP, sides=()) + for fraction in (0.0, 1.1, -0.5): + with pytest.raises(ValueError, match="hp_fraction"): + ScenarioSpec( + name="x", + map_file=MAP, + sides=( + SideSpec( + name="players", + zone="A", + combatant_files=(GOBLIN,), + hp_fraction=fraction, + ), + ), + ) + with pytest.raises(ValueError, match="round_cap"): + ScenarioSpec( + name="x", + map_file=MAP, + sides=(_scenario().sides[0],), + round_cap=0, + ) + + +def test_build_encounter_validation() -> None: + """Shared zone letters, undeclared zones, and empty combatant lists raise.""" + with pytest.raises(ValueError, match="share zone letters"): + build_encounter( + ScenarioSpec( + name="x", + map_file=MAP, + sides=( + SideSpec(name="a", zone="A", combatant_files=(GOBLIN,)), + SideSpec(name="b", zone="A", combatant_files=(ORC,)), + ), + ) + ) + with pytest.raises(ValueError, match="not declared in map deployment"): + build_encounter( + ScenarioSpec( + name="x", + map_file=MAP, + sides=( + SideSpec(name="a", zone="A", combatant_files=(GOBLIN,)), + SideSpec(name="b", zone="C", combatant_files=(ORC,)), + ), + ) + ) + with pytest.raises(ValueError, match="combatant file"): + build_encounter( + ScenarioSpec( + name="x", + map_file=MAP, + sides=( + SideSpec(name="a", zone="A", combatant_files=()), + SideSpec(name="b", zone="B", combatant_files=(ORC,)), + ), + ) + ) + + +def test_variants_default_returns_base() -> None: + """The default hp_fractions=(1.0,) yields exactly one variant equal to base.""" + base = _scenario() + assert scenario_variants(base) == (base,) + + +def test_variants_fractions_names_and_scaling() -> None: + """(0.5, 1.0) yields two variants with suffixed names and scaled HP.""" + base = _scenario() + variants = scenario_variants(base, hp_fractions=(0.5, 1.0)) + + assert len(variants) == 2 + assert variants[0].name == "goblins-vs-orcs hp=0.5" + assert all(side.hp_fraction == 0.5 for side in variants[0].sides) + assert variants[1] == base + + half = build_encounter(variants[0]) + assert half.sides[0].combatants[0].hp_max == 3 + assert half.sides[1].combatants[0].hp_max == 3 + + +def test_variants_empty_fractions_raise() -> None: + """An empty hp_fractions tuple is a ValueError.""" + with pytest.raises(ValueError, match="hp_fractions"): + scenario_variants(_scenario(), hp_fractions=()) + + +def test_variants_invalid_fraction_raises() -> None: + """A fraction outside (0, 1] is a ValueError.""" + with pytest.raises(ValueError, match="hp_fraction"): + scenario_variants(_scenario(), hp_fractions=(1.5,)) + + +def test_variants_deterministic() -> None: + """Two calls with the same inputs produce equal variant tuples.""" + base = _scenario() + fractions = (0.25, 0.5, 0.75, 1.0) + assert scenario_variants(base, hp_fractions=fractions) == scenario_variants( + base, hp_fractions=fractions + ) + + +def test_build_encounter_playable() -> None: + """A compiled encounter runs end-to-end (20 seeded battles).""" + spec = build_encounter(_scenario(goblins=3, orcs=2)) + report = run_encounter(spec, runs=20, seed=1) + assert report.runs == 20 diff --git a/tests/test_phase2_search.py b/tests/test_phase2_search.py new file mode 100644 index 0000000..f02ec05 --- /dev/null +++ b/tests/test_phase2_search.py @@ -0,0 +1,210 @@ +"""Phase-2 strategy search tests (docs/phase-2-llm-architecture.md 10 P3.2/P3.3). + +``evaluate`` measures one strategy on one scenario; ``optimize`` searches the +strategy space (coarse grid, retreat refinement, LLM candidates) and returns a +ranked playbook; ``strategy_name`` renders deterministic labels; and +``format_playbook`` renders the playbook as aligned ASCII columns. These tests +pin the contract: benchmark fields, label rendering, rank ordering, candidate +deduplication, determinism, the ``max_total_runs`` cost bound, and the text +renderer. Small n (runs <= 5) keeps the file fast and deterministic. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from pf1e_simulator.scenarios import ScenarioSpec, SideSpec +from pf1e_simulator.search import evaluate, format_playbook, optimize, strategy_name +from pf1e_simulator.strategy import StrategySpec + +if TYPE_CHECKING: + from pf1e_simulator.search import PlaybookEntry + +ROOT = Path(__file__).resolve().parents[1] +MAP = ROOT / "data" / "maps" / "sample_arena.yaml" +GOBLIN = ROOT / "data" / "monsters" / "goblin.json" +ORC = ROOT / "data" / "monsters" / "orc.json" + +RUNS = 5 +SEED = 7 +ROUND_CAP = 20 +PARTY = "players" + + +def _scenario() -> ScenarioSpec: + return ScenarioSpec( + name="goblins-vs-orcs", + map_file=MAP, + sides=( + SideSpec( + name="players", + zone="A", + combatant_files=(GOBLIN, GOBLIN, GOBLIN), + ), + SideSpec( + name="monsters", + zone="B", + combatant_files=(ORC, ORC), + ), + ), + round_cap=ROUND_CAP, + ) + + +@pytest.fixture(scope="module") +def playbook() -> tuple[PlaybookEntry, ...]: + return optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED) + + +def test_evaluate_default_strategy() -> None: + """evaluate returns a benchmark with sane danger metrics and win band.""" + benchmark = evaluate( + StrategySpec(), + _scenario(), + party=PARTY, + runs=100, + seed=SEED, + ) + + assert benchmark.scenario_name == "goblins-vs-orcs" + assert benchmark.strategy == StrategySpec() + assert benchmark.danger.runs == 100 + assert 0.0 <= benchmark.danger.p_survives <= 1.0 + assert 0.0 <= benchmark.danger.p_tpk <= 1.0 + assert benchmark.danger.expected_rounds > 0.0 + assert set(benchmark.wins) == {"players", "monsters"} + assert benchmark.draws >= 0 + low, high = benchmark.win_rate_band + assert 0.0 <= low <= high <= 1.0 + + +def test_strategy_name() -> None: + """Labels render non-default levers in field order; the name field is excluded.""" + assert strategy_name(StrategySpec()) == "default" + assert strategy_name(StrategySpec(target_priority="weakest_ac")) == "weakest_ac" + assert ( + strategy_name(StrategySpec(target_priority="weakest_ac", spell_policy="none")) + == "weakest_ac, spell_policy=none" + ) + assert ( + strategy_name( + StrategySpec(target_priority="lowest_hp", retreat_hp_fraction=0.5) + ) + == "lowest_hp, retreat=0.5" + ) + assert ( + strategy_name(StrategySpec(name="custom", use_charges=False)) + == "use_charges=false" + ) + + +def test_optimize_small_ranks_and_default(playbook: tuple[PlaybookEntry, ...]) -> None: + """A small optimize returns contiguous ranks ordered by the rank key.""" + assert len(playbook) >= 24 + assert [entry.rank for entry in playbook] == list(range(1, len(playbook) + 1)) + assert any(entry.strategy == StrategySpec() for entry in playbook) + + keys = [ + ( + -entry.benchmark.danger.p_survives, + entry.benchmark.danger.expected_hp_lost, + -entry.benchmark.danger.worst_decile_party_hp, + entry.benchmark.danger.expected_rounds, + strategy_name(entry.strategy), + ) + for entry in playbook + ] + assert keys == sorted(keys) + + +def test_optimize_deterministic() -> None: + """Two optimize calls with the same inputs return identical playbooks.""" + first = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED) + second = optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED) + assert first == second + + +def test_optimize_candidates_deduped() -> None: + """LLM candidates run once each; duplicates and grid duplicates collapse.""" + in_grid = StrategySpec(target_priority="weakest_ac", spell_policy="none") + custom = StrategySpec( + target_priority="lowest_hp", + spell_policy="none", + use_charges=False, + use_class_abilities=False, + retreat_hp_fraction=0.75, + ) + playbook = optimize( + _scenario(), + party=PARTY, + runs=RUNS, + seed=SEED, + candidates=(custom, custom, in_grid), + ) + + custom_matches = [entry for entry in playbook if entry.strategy == custom] + assert len(custom_matches) == 1 + in_grid_matches = [entry for entry in playbook if entry.strategy == in_grid] + assert len(in_grid_matches) == 1 + + direct = evaluate(custom, _scenario(), party=PARTY, runs=RUNS, seed=SEED) + assert custom_matches[0].benchmark.danger == direct.danger + + +def test_budget_raises_below_required_total() -> None: + """A max_total_runs below the required total raises before running.""" + with pytest.raises(ValueError, match="budget") as excinfo: + optimize(_scenario(), party=PARTY, runs=RUNS, seed=SEED, max_total_runs=1) + assert "max_total_runs" in str(excinfo.value) + + +def test_budget_generous_cap_raises_nothing() -> None: + """A generous max_total_runs cap does not raise.""" + playbook = optimize( + _scenario(), + party=PARTY, + runs=RUNS, + seed=SEED, + max_total_runs=10_000, + ) + assert len(playbook) >= 24 + + +def test_format_playbook_columns_and_ascii( + playbook: tuple[PlaybookEntry, ...], +) -> None: + """The renderer shows header tokens, a dash separator, aligned rows, ASCII.""" + text = format_playbook(playbook, PARTY) + + for token in ( + "Rank", + "Strategy", + "Runs", + "P(survive)", + "P(TPK)", + "P(>=1 death)", + "HP lost", + "Worst-decile HP", + "Rounds", + "Win% (3σ)", + ): + assert token in text + + lines = text.splitlines() + assert len(lines) == 2 + len(playbook) + separator = lines[1] + assert separator + assert set(separator) <= {"-", " "} + assert len({len(line) for line in lines}) == 1 + + non_ascii = {char for char in text if ord(char) > 127} + assert non_ascii <= {"σ"} + + top = playbook[0] + rate = top.benchmark.wins.get(PARTY, 0) / top.benchmark.danger.runs + low, high = top.benchmark.win_rate_band + expected = f"{rate * 100:.1f} [{low * 100:.1f}, {high * 100:.1f}]" + assert expected in text