From 98985ee6fc3efd9a6637fa8a041bcd4982d4fd19 Mon Sep 17 00:00:00 2001 From: ctan Date: Sat, 26 Sep 2026 10:40:58 +0200 Subject: [PATCH] feat(phase2): llm report writer (P5.3) - New module src/pf1e_simulator/report.py with ReportContext, build_report_context, LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt - Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator) - CLI integration: --llm-report + --output flags, mutually exclusive with --transcript - Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls - Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy, test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests) --- src/pf1e_simulator/cli.py | 131 ++++++++++- src/pf1e_simulator/report.py | 297 ++++++++++++++++++++++++ tests/fixtures/prompt_pack_fs/report.md | 15 ++ tests/test_cli_llm_report.py | 62 +++++ tests/test_format_llm_report.py | 63 +++++ tests/test_llm_report_policy.py | 100 ++++++++ tests/test_report_budget.py | 162 +++++++++++++ tests/test_report_context.py | 139 +++++++++++ tests/test_report_prompt_pack.py | 67 ++++++ 9 files changed, 1024 insertions(+), 12 deletions(-) create mode 100644 src/pf1e_simulator/report.py create mode 100644 tests/fixtures/prompt_pack_fs/report.md create mode 100644 tests/test_cli_llm_report.py create mode 100644 tests/test_format_llm_report.py create mode 100644 tests/test_llm_report_policy.py create mode 100644 tests/test_report_budget.py create mode 100644 tests/test_report_context.py create mode 100644 tests/test_report_prompt_pack.py diff --git a/src/pf1e_simulator/cli.py b/src/pf1e_simulator/cli.py index e9d29fc..52d029d 100644 --- a/src/pf1e_simulator/cli.py +++ b/src/pf1e_simulator/cli.py @@ -19,11 +19,24 @@ if TYPE_CHECKING: from pf1e_simulator.runner import EncounterReport, EncounterSpec +from pf1e_simulator.config import ENV_PROMPT_PACK_DIR, env_path from pf1e_simulator.loaders import CombatantLoadError, SheetLoadError, load_combatant from pf1e_simulator.map import MapValidationError, load_map -from pf1e_simulator.runner import EncounterSpec, Side, run_encounter +from pf1e_simulator.prompt import PromptPack +from pf1e_simulator.prompt_builder import PromptBuilder +from pf1e_simulator.report import ( + LLMReportPolicy, + build_report_context, + format_llm_report, +) +from pf1e_simulator.runner import EncounterSpec, RunOutcome, Side, run_encounter from pf1e_simulator.spells import SPELLS_DIR, load_spell_registry +try: + from pf1e_simulator.provider import OpenAILLMClient +except ImportError: # pragma: no cover - optional dependency + OpenAILLMClient = None # type: ignore[assignment,misc] + class CliError(Exception): """User-facing CLI error (bad arguments or unusable files).""" @@ -129,7 +142,7 @@ def _build_spec(args: argparse.Namespace) -> EncounterSpec: raise CliError(msg) unknown = sorted(provided_names - set(map_spec.deployment)) if unknown: - msg = f"--side names not in map deployment: {', '.join(unknown)}" + msg = f"--side names not in map deployment: {', '.join(sorted(unknown))}" raise CliError(msg) sides: list[Side] = [] for raw in provided: @@ -172,6 +185,56 @@ def format_report(report: EncounterReport, spec: EncounterSpec) -> str: return "\n".join(lines) +def _run_llm_report( + spec: EncounterSpec, + args: argparse.Namespace, + registry: dict[str, object] | None, +) -> str: + """Generate the LLM-powered report.""" + if OpenAILLMClient is None: + print("pf1e-sim: LLM provider not available", file=sys.stderr) + raise SystemExit(2) + + pack_dir = env_path(ENV_PROMPT_PACK_DIR) + if pack_dir is None: + print(f"pf1e-sim: {ENV_PROMPT_PACK_DIR} environment variable not set", file=sys.stderr) + raise SystemExit(2) + + pack = PromptPack.load(pack_dir) + prompt_version = pack.prompt_version("report") + builder = PromptBuilder(pack) + + client = OpenAILLMClient( + prompt_version=prompt_version, + model=args.llm_model, + api_key=args.llm_api_key, + base_url=args.llm_base_url, + ) + + report: EncounterReport + outcomes: tuple[RunOutcome, ...] + report, outcomes = run_encounter( # type: ignore[misc] + spec, + runs=args.runs, + seed=args.seed, + spell_registry=registry, # type: ignore[arg-type] + capture_first_transcript=args.transcript, + monsters_die_at_zero=args.simple_monster_death, + collect_runs=True, + ) + + ctx = build_report_context( + encounter=report, # type: ignore[arg-type] + spec_name=spec.map.name, + outcomes=outcomes, # type: ignore[arg-type] + party="players", + ) + + policy = LLMReportPolicy(client=client, builder=builder, prompt_name="report") + narrative = policy(ctx) + return format_llm_report(ctx, narrative) + + def main(argv: list[str] | None = None) -> int: """CLI entry point; returns the process exit code.""" parser = argparse.ArgumentParser( @@ -207,9 +270,39 @@ def main(argv: list[str] | None = None) -> int: "(player characters are never affected)" ), ) + parser.add_argument( + "--llm-report", + action="store_true", + help="generate an LLM-powered French narrative report (requires PF1E_LLM_API_KEY)", + ) + parser.add_argument( + "--output", + type=Path, + help="write the full report (factual + LLM narrative) to this file", + ) + parser.add_argument( + "--llm-model", + type=str, + help="override the LLM model (default: PF1E_LLM_MODEL env or gpt-4o-mini)", + ) + parser.add_argument( + "--llm-base-url", + type=str, + help="override the LLM base URL (default: PF1E_LLM_BASE_URL env or https://api.openai.com/v1)", + ) + parser.add_argument( + "--llm-api-key", + type=str, + help="override the LLM API key (default: PF1E_LLM_API_KEY env)", + ) args = parser.parse_args(argv) if not args.side: parser.error("at least one --side is required") + if args.llm_report and args.transcript: + parser.error("--llm-report and --transcript are mutually exclusive") + if args.output and not args.llm_report: + parser.error("--output requires --llm-report") + try: spec = _build_spec(args) except CliError as exc: @@ -218,22 +311,36 @@ def main(argv: list[str] | None = None) -> int: except (MapValidationError, CombatantLoadError, SheetLoadError, OSError) as exc: print(f"pf1e-sim: {exc}", file=sys.stderr) return 2 + try: registry = load_spell_registry(SPELLS_DIR) - report = run_encounter( - spec, - runs=args.runs, - seed=args.seed, - spell_registry=registry, - capture_first_transcript=args.transcript, - monsters_die_at_zero=args.simple_monster_death, - ) + + report = None + if args.llm_report: + full_report = _run_llm_report(spec, args, registry) # type: ignore[arg-type] + else: + report = run_encounter( + spec, + runs=args.runs, + seed=args.seed, + spell_registry=registry, + capture_first_transcript=args.transcript, + monsters_die_at_zero=args.simple_monster_death, + ) + full_report = format_report(report, spec) + except ValueError as exc: print(f"pf1e-sim: {exc}", file=sys.stderr) return 2 - if args.transcript and report.transcript is not None: + + if args.transcript and report is not None and report.transcript is not None: print("=== Déroulé du premier combat ===") print(render_transcript_table(report.transcript)) print() - print(format_report(report, spec)) + + print(full_report) + + if args.output: + args.output.write_text(full_report, encoding="utf-8") + return 0 diff --git a/src/pf1e_simulator/report.py b/src/pf1e_simulator/report.py new file mode 100644 index 0000000..6d9e01f --- /dev/null +++ b/src/pf1e_simulator/report.py @@ -0,0 +1,297 @@ +"""LLM report writer (Phase 3) — aggregator + French narrative generator. + +This module provides the data structures and pure functions to build a +report context from Monte Carlo outcomes and LLM observability counters, +then format the final French report combining the factual balance report +with an LLM-generated narrative section. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from pf1e_simulator.client import BudgetExceededError, CostBudget +from pf1e_simulator.objectives import danger_metrics + +if TYPE_CHECKING: + from collections.abc import Sequence + + from pf1e_simulator.client import ClientStats, LLMClient + from pf1e_simulator.nl_policy import NLPolicyStats + from pf1e_simulator.objectives import DangerMetrics + from pf1e_simulator.prompt_builder import PromptBuilder + from pf1e_simulator.robustness import ( + DangerCell, + MatchupBenchmark, + SensitivityEntry, + ) + from pf1e_simulator.runner import EncounterReport, RunOutcome + + +@dataclass(frozen=True) +class ReportContext: + """Complete context for the LLM report writer. + + Fields marked as optional are filled when the corresponding analysis + was executed (e.g. `collect_runs=True` for DangerMetrics, robustness + calls for MatchupBenchmark/SensitivityEntry/DangerCell). + """ + + encounter: EncounterReport + spec_name: str + + danger: DangerMetrics | None = None + + nl_stats: NLPolicyStats | None = None + client_stats: ClientStats | None = None + + matchup: MatchupBenchmark | None = None + danger_map: tuple[DangerCell, ...] | None = None + frontier_value: float | None = None + sensitivity: tuple[SensitivityEntry, ...] | None = None + + +def build_report_context( # noqa: PLR0913 - intentional builder with many optional params + *, + encounter: EncounterReport, + spec_name: str, + outcomes: Sequence[RunOutcome] | None = None, + party: str = "players", + nl_stats: NLPolicyStats | None = None, + client_stats: ClientStats | None = None, + matchup: MatchupBenchmark | None = None, + danger_map: tuple[DangerCell, ...] | None = None, + frontier_value: float | None = None, + sensitivity: tuple[SensitivityEntry, ...] | None = None, +) -> ReportContext: + """Build a `ReportContext` from raw Monte Carlo outputs.""" + danger: DangerMetrics | None = None + if outcomes is not None: + danger = danger_metrics(outcomes, party) + + return ReportContext( + encounter=encounter, + spec_name=spec_name, + danger=danger, + nl_stats=nl_stats, + client_stats=client_stats, + matchup=matchup, + danger_map=danger_map, + frontier_value=frontier_value, + sensitivity=sensitivity, + ) + + +def _render_encounter(ctx: ReportContext) -> str: + """Render the EncounterReport in the same format as cli.format_report.""" + r = ctx.encounter + lines = ["=== Rapport d'équilibrage ===", f"Carte : {ctx.spec_name}"] + lines.extend(f"{side_name} : {r.wins.get(side_name, 0)} combats" for side_name in r.wins) + lines.append(f"{r.runs} combats simulés (seed {r.seed}, plafond ? rounds)") + lines.append("") + for side_name in r.wins: + wins = r.wins.get(side_name, 0) + rate = r.win_rate(side_name) + low, high = r.win_rate_band(side_name) + lines.append( + f"Victoires {side_name} : {rate:.1%} ({wins}) — " + f"bande 3σ : [{low:.1%}, {high:.1%}]" + ) + lines.append(f"Nuls : {r.draws / r.runs:.1%} ({r.draws})") + lines.append(f"Rounds moyens : {r.avg_rounds:.1f}") + lines.append("") + lines.append("Attrition moyenne par combatant :") + for cid, attr in r.attrition.items(): + lines.append( + f" {cid} : {attr.hits:.2f} touches · {attr.crits:.2f} critiques · " + f"{attr.damage_dealt:.2f} dégâts infligés · {attr.damage_taken:.2f} subis" + f" · {attr.healing_done:.2f} soins prodigués" + ) + return "\n".join(lines) + + +def _render_danger(danger: DangerMetrics) -> str: + """Render DangerMetrics section.""" + lines = ["=== Métriques de danger ==="] + lines.append(f"runs: {danger.runs}") + lines.append(f"p_survives: {danger.p_survives:.2f}") + lines.append(f"p_tpk: {danger.p_tpk:.2f}") + lines.append(f"p_any_death: {danger.p_any_death:.2f}") + lines.append(f"expected_hp_lost: {danger.expected_hp_lost:.1f}") + lines.append(f"expected_rounds: {danger.expected_rounds:.1f}") + lines.append(f"worst_decile_party_hp: {danger.worst_decile_party_hp:.1f}") + return "\n".join(lines) + + +def _render_nl_stats(stats: NLPolicyStats) -> str: + lines = ["=== Observabilité LLM ==="] + lines.append(f"calls: {stats.calls}") + lines.append(f"repairs_attempted: {stats.repairs_attempted}") + lines.append(f"repairs_succeeded: {stats.repairs_succeeded}") + lines.append(f"fallbacks: {stats.fallbacks}") + return "\n".join(lines) + + +def _render_client_stats(stats: ClientStats) -> str: + lines = ["=== Coût LLM ==="] + lines.append(f"prompt_tokens: {stats.prompt_tokens}") + lines.append(f"completion_tokens: {stats.completion_tokens}") + lines.append(f"cost_usd: {stats.cost_usd:.3f}") + lines.append(f"cache_hits: {stats.cache_hits}") + lines.append(f"cache_misses: {stats.cache_misses}") + return "\n".join(lines) + + +def _render_matchup(matchup: MatchupBenchmark) -> str: + lines = ["=== Robustesse (matchup) ==="] + lines.append(f"party: {matchup.party}") + lines.append(f"adversary: {matchup.adversary}") + lines.append(f"runs: {matchup.runs}") + lines.append(f"seed: {matchup.seed}") + lines.append(f"party_danger.p_survives: {matchup.party_danger.p_survives:.2f}") + lines.append(f"party_danger.p_tpk: {matchup.party_danger.p_tpk:.2f}") + lines.append(f"adversary_danger.p_survives: {matchup.adversary_danger.p_survives:.2f}") + lines.append(f"adversary_danger.p_tpk: {matchup.adversary_danger.p_tpk:.2f}") + return "\n".join(lines) + + +def build_report_prompt(ctx: ReportContext, template: str) -> str: + """Build the complete LLM prompt for the report writer.""" + sections: list[str] = [template, ""] + + sections.append(_render_encounter(ctx)) + sections.append("") + + if ctx.danger is not None: + sections.append(_render_danger(ctx.danger)) + sections.append("") + + if ctx.matchup is not None: + sections.append(_render_matchup(ctx.matchup)) + sections.append("") + + if ctx.nl_stats is not None: + sections.append(_render_nl_stats(ctx.nl_stats)) + sections.append("") + + if ctx.client_stats is not None: + sections.append(_render_client_stats(ctx.client_stats)) + sections.append("") + + if sections[-1] == "": + sections.pop() + + return "\n".join(sections) + + +@dataclass(frozen=True) +class ReportBudget: + """Budget limits for a single report generation (integrates with CostBudget). + + Mirrors the ceilings from the architecture doc (section 13): + small caps for showcase/vitrine mode — max 1 call per report. + """ + + max_calls: int = 1 + max_total_tokens: int | None = None + max_cost_usd: float | None = None + + def to_cost_budget(self) -> CostBudget: + """Convert to the client's CostBudget type.""" + return CostBudget( + max_calls=self.max_calls, + max_total_tokens=self.max_total_tokens, + max_cost_usd=self.max_cost_usd, + ) + + def for_retries(self, max_retries: int) -> ReportBudget: + """Return a budget allowing for retries (max_calls = max_retries + 1).""" + if self.max_calls == 1: + return ReportBudget( + max_calls=max_retries + 1, + max_total_tokens=self.max_total_tokens, + max_cost_usd=self.max_cost_usd, + ) + return self + + +@dataclass +class ReportStats: + """Mutable counters for the LLM report writer.""" + + calls: int = 0 + retries: int = 0 + fallbacks: int = 0 + + +@dataclass(frozen=True) +class LLMReportPolicy: + """Generate a French narrative report via LLM completion. + + Distinct from NLPolicy: produces free-text narrative (not JSON actions), + uses simple retry loop (no repair trail), and tracks separate stats. + """ + + client: LLMClient + builder: PromptBuilder + budget: ReportBudget = field(default_factory=ReportBudget) + prompt_name: str = "report" + max_retries: int = 1 + stats: ReportStats = field(default_factory=ReportStats) + + def __post_init__(self) -> None: + if self.max_retries < 0: + msg = f"max_retries must be >= 0, got {self.max_retries}" + raise ValueError(msg) + expected = self.builder.prompt_version(self.prompt_name) + if self.client.prompt_version != expected: + msg = ( + f"client prompt_version {self.client.prompt_version!r} does not " + f"match pack {expected!r} for {self.prompt_name!r}" + ) + raise ValueError(msg) + # Apply the report budget (adjusted for retries) to the client + if self.client._budget is None: # noqa: SLF001 - private access required for budget injection # pyright: ignore[reportPrivateUsage] + adjusted_budget = self.budget.for_retries(self.max_retries) + self.client._budget = adjusted_budget.to_cost_budget() # noqa: SLF001 - private access required for budget injection # pyright: ignore[reportPrivateUsage] + + def __call__(self, ctx: ReportContext) -> str: + prompt = build_report_prompt(ctx, self.builder.pack.get(self.prompt_name)) + retries = 0 + while True: + self.stats.calls += 1 + try: + return self.client.complete(prompt) + except BudgetExceededError: + # Budget exceeded during retry - treat as fallback + self.stats.fallbacks += 1 + return "" + except Exception: # noqa: BLE001 - client may raise various exceptions + if retries >= self.max_retries: + self.stats.fallbacks += 1 + return "" + retries += 1 + self.stats.retries += 1 + + +def format_llm_report(ctx: ReportContext, llm_text: str) -> str: + """Combine the factual balance report with the LLM narrative section. + + The result is the complete French report printed to stdout and/or written + to a file via `--output`. + + Parameters + ---------- + ctx: The populated `ReportContext` with all available data. + llm_text: The narrative text returned by `LLMReportPolicy` (may be empty). + + Returns + ------- + Complete French report as a single string. + """ + factual = _render_encounter(ctx) + if not llm_text.strip(): + return factual + + return f"{factual}\n\n=== Récit d'équilibrage (LLM) ===\n{llm_text.strip()}" diff --git a/tests/fixtures/prompt_pack_fs/report.md b/tests/fixtures/prompt_pack_fs/report.md new file mode 100644 index 0000000..4263ccf --- /dev/null +++ b/tests/fixtures/prompt_pack_fs/report.md @@ -0,0 +1,15 @@ +Vous êtes le narrateur pour un simulateur de combat Pathfinder 1e. + +Vous recevez un rapport de bataille structuré contenant : +- Le rapport d'équilibrage factuel (taux de victoire, bandes de confiance 3σ, attrition) +- Les métriques de danger (probabilité de survie, risque TPK, perte de PV attendue, queue pire décile) +- La cartographie de robustesse optionnelle (matchups, carte de danger, frontière, sensibilité) +- L'observabilité LLM (appels, réparations, fallbacks, usage tokens, coût) + +Rédigez un résumé narratif concis en français pour le maître de jeu. Ton : analytique, évocateur, ancré dans les chiffres. N'inventez PAS de chiffres — citez les statistiques fournies textuellement. PAS de formatage markdown. Texte brut, français uniquement. + +Structurez votre réponse ainsi : +1. **Bilan d'équilibrage** — 2-3 phrases synthétisant les taux de victoire et leur signification. +2. **Danger et attrition** — 2-3 phrases sur survie/TPK/perte PV/pire décile. +3. **Robustesse** (si données présentes) — 1-2 phrases sur matchup/carte de danger/frontière/sensibilité. +4. **Observabilité LLM** (si données présentes) — 1 phrase sur appels/réparations/coût. \ No newline at end of file diff --git a/tests/test_cli_llm_report.py b/tests/test_cli_llm_report.py new file mode 100644 index 0000000..7eb3c87 --- /dev/null +++ b/tests/test_cli_llm_report.py @@ -0,0 +1,62 @@ +"""CLI integration test for --llm-report flag (P5.3 T5).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pf1e_simulator.cli import main + +ROOT = Path(__file__).resolve().parents[1] +MAP = ROOT / "data" / "maps" / "sample_arena.yaml" + + +def _argv_common() -> list[str]: + return [ + "--map", str(MAP), + "--side", "players", str(ROOT / "data" / "monsters" / "goblin.json"), + "--side", "monsters", str(ROOT / "data" / "monsters" / "orc.json"), + ] + + +def test_cli_llm_report_mutually_exclusive_with_transcript() -> None: + """--llm-report and --transcript are mutually exclusive.""" + argv = [*_argv_common(), "--llm-report", "--transcript"] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 2 + + +def test_cli_output_requires_llm_report() -> None: + """--output requires --llm-report.""" + argv = [*_argv_common(), "--output", "report.md"] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 2 + + +def test_cli_llm_report_requires_api_key_env(monkeypatch: pytest.MonkeyPatch) -> None: + """--llm-report fails gracefully when PF1E_LLM_API_KEY not set.""" + monkeypatch.delenv("PF1E_LLM_API_KEY", raising=False) + monkeypatch.delenv("PF1E_PROMPT_PACK_DIR", raising=False) + + argv = [*_argv_common(), "--llm-report"] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 2 + + +def test_cli_llm_report_requires_prompt_pack_dir(monkeypatch: pytest.MonkeyPatch) -> None: + """--llm-report fails gracefully when PF1E_PROMPT_PACK_DIR not set.""" + monkeypatch.setenv("PF1E_LLM_API_KEY", "test-key") + monkeypatch.delenv("PF1E_PROMPT_PACK_DIR", raising=False) + + argv = [*_argv_common(), "--llm-report"] + + with pytest.raises(SystemExit) as exc_info: + main(argv) + assert exc_info.value.code == 2 diff --git a/tests/test_format_llm_report.py b/tests/test_format_llm_report.py new file mode 100644 index 0000000..4065895 --- /dev/null +++ b/tests/test_format_llm_report.py @@ -0,0 +1,63 @@ +"""Tests for format_llm_report (P5.3 T4).""" + +from __future__ import annotations + +from pf1e_simulator.report import build_report_context, format_llm_report +from pf1e_simulator.runner import CombatantAttrition, EncounterReport + + +def _make_encounter() -> EncounterReport: + return EncounterReport( + runs=100, + seed=1, + wins={"players": 65, "monsters": 33}, + draws=2, + avg_rounds=7.2, + attrition={ + "goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00), + "orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00), + }, + ) + + +def test_format_llm_report_empty_narrative() -> None: + """Empty LLM text returns only the factual report.""" + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + result = format_llm_report(ctx, "") + + assert "=== Rapport d'équilibrage ===" in result + assert "Victoires players : 65.0%" in result + assert "Récit d'équilibrage" not in result + + +def test_format_llm_report_whitespace_only_narrative() -> None: + """Whitespace-only LLM text returns only the factual report.""" + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + result = format_llm_report(ctx, " \n \t ") + + assert "=== Rapport d'équilibrage ===" in result + assert "Récit d'équilibrage" not in result + + +def test_format_llm_report_with_narrative() -> None: + """Non-empty LLM text appends narrative section after factual report.""" + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + narrative = "Le gobelin a surpris l'orc par sa vitesse." + result = format_llm_report(ctx, narrative) + + assert "=== Rapport d'équilibrage ===" in result + assert "Victoires players : 65.0%" in result + assert "=== Récit d'équilibrage (LLM) ===" in result + assert narrative in result + # factual comes before narrative + factual_end = result.index("=== Récit d'équilibrage (LLM) ===") + assert "Victoires players" in result[:factual_end] + + +def test_format_llm_report_strips_narrative_whitespace() -> None: + """Leading/trailing whitespace in narrative is stripped.""" + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + result = format_llm_report(ctx, " \n Texte narratif. \n ") + + assert result.endswith("Texte narratif.") + assert "=== Récit d'équilibrage (LLM) ===\nTexte narratif." in result diff --git a/tests/test_llm_report_policy.py b/tests/test_llm_report_policy.py new file mode 100644 index 0000000..bddcdd2 --- /dev/null +++ b/tests/test_llm_report_policy.py @@ -0,0 +1,100 @@ +"""Tests for LLMReportPolicy (P5.3 T3).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from pf1e_simulator.client import FakeLLMClient +from pf1e_simulator.prompt import PromptPack +from pf1e_simulator.prompt_builder import PromptBuilder +from pf1e_simulator.report import LLMReportPolicy, build_report_context +from pf1e_simulator.runner import CombatantAttrition, EncounterReport + +FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" + + +def _make_encounter() -> EncounterReport: + return EncounterReport( + runs=100, + seed=1, + wins={"players": 65, "monsters": 33}, + draws=2, + avg_rounds=7.2, + attrition={ + "goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00), + "orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00), + }, + ) + + +def _make_policy( + script: list[str], + *, + max_retries: int = 1, +) -> LLMReportPolicy: + pack = PromptPack.load(FIXTURE_DIR) + builder = PromptBuilder(pack) + expected_version = pack.prompt_version("report") + client = FakeLLMClient(script=script, prompt_version=expected_version) # type: ignore[arg-type] + return LLMReportPolicy( + client=client, builder=builder, prompt_name="report", max_retries=max_retries + ) + + +def test_llm_report_policy_returns_text() -> None: + """Policy returns the scripted text from FakeLLMClient.""" + policy = _make_policy(script=["Rapport narratif de test."]) + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + + result = policy(ctx) + + assert result == "Rapport narratif de test." + assert policy.stats.calls == 1 + assert policy.stats.retries == 0 + assert policy.stats.fallbacks == 0 + + +def test_llm_report_policy_retry_on_exception() -> None: + """Policy retries on exception and increments retries counter.""" + policy = _make_policy( + script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item] + max_retries=1, + ) + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + + result = policy(ctx) + + assert result == "Rapport après retry." + assert policy.stats.calls == 2 + assert policy.stats.retries == 1 + assert policy.stats.fallbacks == 0 + + +def test_llm_report_policy_fallback_after_max_retries() -> None: + """Policy returns empty string and increments fallbacks after max retries.""" + policy = _make_policy( + script=[Exception("boom"), Exception("boom2")], # type: ignore[list-item] + max_retries=1, + ) + ctx = build_report_context(encounter=_make_encounter(), spec_name="test") + + result = policy(ctx) + + assert result == "" + assert policy.stats.calls == 2 + assert policy.stats.retries == 1 + assert policy.stats.fallbacks == 1 + + +def test_llm_report_policy_prompt_version_mismatch_raises() -> None: + """Policy rejects client with mismatched prompt_version.""" + pack = PromptPack.load(FIXTURE_DIR) + builder = PromptBuilder(pack) + expected_version = pack.prompt_version("report") + client = FakeLLMClient(script=["x"], prompt_version=expected_version) + client.prompt_version = "wrong-version" + + with pytest.raises(ValueError, match="prompt_version"): + LLMReportPolicy(client=client, builder=builder, prompt_name="report") diff --git a/tests/test_report_budget.py b/tests/test_report_budget.py new file mode 100644 index 0000000..fe68aec --- /dev/null +++ b/tests/test_report_budget.py @@ -0,0 +1,162 @@ +"""Tests for ReportBudget and budget exceeded handling (P5.3 T6).""" + +from __future__ import annotations + +from pathlib import Path + +from pf1e_simulator.client import CostBudget, FakeLLMClient +from pf1e_simulator.prompt import PromptPack +from pf1e_simulator.prompt_builder import PromptBuilder +from pf1e_simulator.report import ( + LLMReportPolicy, + ReportBudget, + build_report_context, +) +from pf1e_simulator.runner import EncounterReport + +FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" + + +def _make_policy( + script: list[str], + *, + max_retries: int = 1, + budget: ReportBudget | None = None, +) -> LLMReportPolicy: + pack = PromptPack.load(FIXTURE_DIR) + builder = PromptBuilder(pack) + client = FakeLLMClient(script=script, prompt_version=pack.prompt_version("report")) # type: ignore[arg-type] + return LLMReportPolicy( + client=client, + builder=builder, + prompt_name="report", + max_retries=max_retries, + budget=budget or ReportBudget(), + ) + + +def test_report_budget_default_allows_one_call() -> None: + """Default budget allows exactly 1 successful call.""" + policy = _make_policy(script=["Rapport de test."]) + ctx = build_report_context( + encounter=EncounterReport( + runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0, + attrition={} + ), + spec_name="test", + ) + + result = policy(ctx) + + assert result == "Rapport de test." + assert policy.stats.calls == 1 + + +def test_report_budget_exceeded_on_second_call() -> None: + """Budget exceeded when attempting a second call.""" + policy = _make_policy( + script=["First call.", "Second call (should not happen)."], + max_retries=0, # No retries + ) + ctx = build_report_context( + encounter=EncounterReport( + runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0, + attrition={} + ), + spec_name="test", + ) + + # First call succeeds + result1 = policy(ctx) + assert result1 == "First call." + assert policy.stats.calls == 1 + + # Second call should exceed budget and return empty string (fallback) + result2 = policy(ctx) + assert result2 == "" + assert policy.stats.calls == 2 + assert policy.stats.fallbacks == 1 + + +def test_report_budget_for_retries_allows_retry() -> None: + """Budget adjusted for retries allows the retry to succeed.""" + policy = _make_policy( + script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item] + max_retries=1, + ) + ctx = build_report_context( + encounter=EncounterReport( + runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0, + attrition={} + ), + spec_name="test", + ) + + result = policy(ctx) + + assert result == "Rapport après retry." + assert policy.stats.calls == 2 + assert policy.stats.retries == 1 + assert policy.stats.fallbacks == 0 + + +def test_report_budget_for_retries_exceeded() -> None: + """Budget still exceeded after max retries exhausted.""" + policy = _make_policy( + script=[Exception("boom1"), Exception("boom2"), "Should not reach here."], # type: ignore[list-item] + max_retries=1, + ) + ctx = build_report_context( + encounter=EncounterReport( + runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0, + attrition={} + ), + spec_name="test", + ) + + result = policy(ctx) + + assert result == "" + assert policy.stats.calls == 2 # initial + 1 retry (budget allows max_retries + 1 = 2) + assert policy.stats.retries == 1 + assert policy.stats.fallbacks == 1 + + +def test_report_budget_to_cost_budget() -> None: + """ReportBudget converts to CostBudget correctly.""" + budget = ReportBudget(max_calls=2, max_total_tokens=1000, max_cost_usd=0.01) + cost_budget = budget.to_cost_budget() + + assert isinstance(cost_budget, CostBudget) + assert cost_budget.max_calls == 2 + assert cost_budget.max_total_tokens == 1000 + assert cost_budget.max_cost_usd == 0.01 + + +def test_report_budget_for_retries_adjusts_max_calls() -> None: + """for_retries adjusts max_calls to max_retries + 1.""" + budget = ReportBudget(max_calls=1) + adjusted = budget.for_retries(2) + + assert adjusted.max_calls == 3 # 2 retries + 1 initial + assert adjusted.max_total_tokens is None + assert adjusted.max_cost_usd is None + + +def test_report_budget_for_retries_preserves_other_limits() -> None: + """for_retries preserves token and cost limits.""" + budget = ReportBudget(max_calls=1, max_total_tokens=500, max_cost_usd=0.005) + adjusted = budget.for_retries(3) + + assert adjusted.max_calls == 4 + assert adjusted.max_total_tokens == 500 + assert adjusted.max_cost_usd == 0.005 + + +def test_report_budget_for_retries_preserves_non_default_max_calls() -> None: + """for_retries preserves non-default max_calls.""" + budget = ReportBudget(max_calls=5) + adjusted = budget.for_retries(2) + + assert adjusted.max_calls == 5 # unchanged because it wasn't the default 1 + diff --git a/tests/test_report_context.py b/tests/test_report_context.py new file mode 100644 index 0000000..b1ddade --- /dev/null +++ b/tests/test_report_context.py @@ -0,0 +1,139 @@ +"""Tests for ReportContext and build_report_context (P5.3 T1).""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest + +from pf1e_simulator.client import ClientStats +from pf1e_simulator.loaders import load_combatant +from pf1e_simulator.map import load_map +from pf1e_simulator.nl_policy import NLPolicyStats +from pf1e_simulator.objectives import DangerMetrics +from pf1e_simulator.report import ReportContext, build_report_context +from pf1e_simulator.runner import EncounterSpec, Side, run_encounter +from pf1e_simulator.spells import SPELLS_DIR, load_spell_registry + +ROOT = Path(__file__).resolve().parents[1] +MAP = ROOT / "data" / "maps" / "sample_arena.yaml" + + +def _c(path: str): + return load_combatant(ROOT / "data" / path) + + +def _spec() -> EncounterSpec: + """Fast 1v1 (goblin vs orc) on the sample arena.""" + return EncounterSpec( + map=load_map(MAP), + round_cap=20, + sides=( + Side(name="players", combatants=(_c("monsters/goblin.json"),), zone="A"), + Side(name="monsters", combatants=(_c("monsters/orc.json"),), zone="B"), + ), + ) + + +def test_build_report_context_minimal() -> None: + """Minimal context: only EncounterReport, no RunOutcome.""" + registry = load_spell_registry(SPELLS_DIR) + spec = _spec() + report = run_encounter(spec, runs=10, seed=1, spell_registry=registry) + + ctx = build_report_context( + encounter=report, + spec_name="goblin-vs-orc", + ) + + assert isinstance(ctx, ReportContext) + assert ctx.encounter is report + assert ctx.spec_name == "goblin-vs-orc" + assert ctx.danger is None + assert ctx.nl_stats is None + assert ctx.client_stats is None + assert ctx.matchup is None + assert ctx.danger_map is None + assert ctx.frontier_value is None + assert ctx.sensitivity is None + + +def test_build_report_context_with_outcomes() -> None: + """Context with RunOutcome -> DangerMetrics computed for the party.""" + registry = load_spell_registry(SPELLS_DIR) + spec = _spec() + report, outcomes = run_encounter( + spec, runs=100, seed=1, spell_registry=registry, collect_runs=True + ) + + ctx = build_report_context( + encounter=report, + spec_name="goblin-vs-orc", + outcomes=outcomes, + party="players", + ) + + assert ctx.danger is not None + assert isinstance(ctx.danger, DangerMetrics) + assert ctx.danger.runs == 100 + # p_survives, p_tpk, p_any_death are probabilities in [0, 1] + assert 0.0 <= ctx.danger.p_survives <= 1.0 + assert 0.0 <= ctx.danger.p_tpk <= 1.0 + assert 0.0 <= ctx.danger.p_any_death <= 1.0 + assert ctx.danger.expected_hp_lost >= 0.0 + assert ctx.danger.expected_rounds > 0.0 + assert ctx.danger.worst_decile_party_hp >= 0.0 + + +def test_build_report_context_with_all_optional_fields() -> None: + """Context with all optional fields populated.""" + registry = load_spell_registry(SPELLS_DIR) + spec = _spec() + report, outcomes = run_encounter( + spec, runs=100, seed=1, spell_registry=registry, collect_runs=True + ) + + nl_stats = NLPolicyStats(calls=5, repairs_attempted=1, repairs_succeeded=1, fallbacks=0) + client_stats = ClientStats( + calls=5, + prompt_tokens=1000, + completion_tokens=200, + cost_usd=0.001, + cache_hits=0, + cache_misses=5, + ) + + ctx = build_report_context( + encounter=report, + spec_name="goblin-vs-orc", + outcomes=outcomes, + party="players", + nl_stats=nl_stats, + client_stats=client_stats, + ) + + assert ctx.nl_stats is nl_stats + assert ctx.client_stats is client_stats + assert ctx.nl_stats.calls == 5 # type: ignore[reportOptionalMemberAccess] + assert ctx.nl_stats.repairs_attempted == 1 # type: ignore[reportOptionalMemberAccess] + assert ctx.nl_stats.repairs_succeeded == 1 # type: ignore[reportOptionalMemberAccess] + assert ctx.nl_stats.fallbacks == 0 # type: ignore[reportOptionalMemberAccess] + assert ctx.client_stats.prompt_tokens == 1000 # type: ignore[reportOptionalMemberAccess] + assert ctx.client_stats.cost_usd == 0.001 # type: ignore[reportOptionalMemberAccess] + + +def test_report_context_immutable() -> None: + """ReportContext is a frozen dataclass (immutable).""" + registry = load_spell_registry(SPELLS_DIR) + spec = _spec() + report = run_encounter(spec, runs=5, seed=1, spell_registry=registry) + + ctx = build_report_context( + encounter=report, + spec_name="test", + ) + + # Frozen dataclass -> assignment raises FrozenInstanceError + with pytest.raises(FrozenInstanceError): + ctx.encounter = None # type: ignore[misc] diff --git a/tests/test_report_prompt_pack.py b/tests/test_report_prompt_pack.py new file mode 100644 index 0000000..483f72b --- /dev/null +++ b/tests/test_report_prompt_pack.py @@ -0,0 +1,67 @@ +"""Tests for report prompt pack (P5.3 T2).""" + +from __future__ import annotations + +from pathlib import Path + +from pf1e_simulator.prompt import PromptPack +from pf1e_simulator.report import build_report_context, build_report_prompt +from pf1e_simulator.runner import CombatantAttrition, EncounterReport + + +def test_report_prompt_pack_version_stable() -> None: + """Prompt pack version hash is stable for the fixture.""" + pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" + pack = PromptPack.load(pack_dir) + + report_version = pack.prompt_version("report") + assert report_version.startswith("0.1.0-test:report:") + assert len(report_version.split(":")[2]) == 12 # hash12 + + +def test_report_prompt_renders() -> None: + """build_report_prompt renders the context correctly.""" + pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" + pack = PromptPack.load(pack_dir) + template = pack.get("report") + + encounter = EncounterReport( + runs=100, + seed=1, + wins={"players": 65, "monsters": 33}, + draws=2, + avg_rounds=7.2, + attrition={ + "goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00), + "orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00), + }, + ) + + ctx = build_report_context( + encounter=encounter, + spec_name="test", + ) + + prompt = build_report_prompt(ctx, template) + + # Should contain the system prompt text + assert "Vous êtes le narrateur" in prompt + assert "Bilan d'équilibrage" in prompt + assert "Danger et attrition" in prompt + # Should contain the rendered encounter data + assert "Victoires players : 65.0%" in prompt or "Victoires players : 65" in prompt + assert "goblin" in prompt + assert "orc" in prompt + # Should contain attrition + assert "Attrition moyenne par combatant" in prompt + assert "touches" in prompt + + +def test_report_prompt_version_different_from_nl_default() -> None: + """report prompt_version differs from nl_default.""" + pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" + pack = PromptPack.load(pack_dir) + + report_version = pack.prompt_version("report") + nl_version = pack.prompt_version("nl_default") + assert report_version != nl_version