ctan/pf1e-simulator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(phase2): llm report writer (P5.3)

Browse Source 
- New module src/pf1e_simulator/report.py with ReportContext, build_report_context,
  LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt
- Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator)
- CLI integration: --llm-report + --output flags, mutually exclusive with --transcript
- Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls
- Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy,
  test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
This commit is contained in:
Thien An
2026-09-26 10:40:58 +02:00
parent c454a67b84
commit 98985ee6fc
9 changed files with 1024 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/prompt_pack_fs/report.md
Vendored
+15
View File
@@ -0,0 +1,15 @@
Vous êtes le narrateur pour un simulateur de combat Pathfinder 1e.
Vous recevez un rapport de bataille structuré contenant :
- Le rapport d'équilibrage factuel (taux de victoire, bandes de confiance 3σ, attrition)
- Les métriques de danger (probabilité de survie, risque TPK, perte de PV attendue, queue pire décile)
- La cartographie de robustesse optionnelle (matchups, carte de danger, frontière, sensibilité)
- L'observabilité LLM (appels, réparations, fallbacks, usage tokens, coût)
Rédigez un résumé narratif concis en français pour le maître de jeu. Ton : analytique, évocateur, ancré dans les chiffres. N'inventez PAS de chiffres — citez les statistiques fournies textuellement. PAS de formatage markdown. Texte brut, français uniquement.
Structurez votre réponse ainsi :
1. **Bilan d'équilibrage** — 2-3 phrases synthétisant les taux de victoire et leur signification.
2. **Danger et attrition** — 2-3 phrases sur survie/TPK/perte PV/pire décile.
3. **Robustesse** (si données présentes) — 1-2 phrases sur matchup/carte de danger/frontière/sensibilité.
4. **Observabilité LLM** (si données présentes) — 1 phrase sur appels/réparations/coût.
tests/test_cli_llm_report.py
+62
View File
@@ -0,0 +1,62 @@
"""CLI integration test for --llm-report flag (P5.3 T5)."""
from __future__ import annotations
from pathlib import Path
import pytest
from pf1e_simulator.cli import main
ROOT = Path(__file__).resolve().parents[1]
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
def _argv_common() -> list[str]:
return [
"--map", str(MAP),
"--side", "players", str(ROOT / "data" / "monsters" / "goblin.json"),
"--side", "monsters", str(ROOT / "data" / "monsters" / "orc.json"),
]
def test_cli_llm_report_mutually_exclusive_with_transcript() -> None:
"""--llm-report and --transcript are mutually exclusive."""
argv = [*_argv_common(), "--llm-report", "--transcript"]
with pytest.raises(SystemExit) as exc_info:
main(argv)
assert exc_info.value.code == 2
def test_cli_output_requires_llm_report() -> None:
"""--output requires --llm-report."""
argv = [*_argv_common(), "--output", "report.md"]
with pytest.raises(SystemExit) as exc_info:
main(argv)
assert exc_info.value.code == 2
def test_cli_llm_report_requires_api_key_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""--llm-report fails gracefully when PF1E_LLM_API_KEY not set."""
monkeypatch.delenv("PF1E_LLM_API_KEY", raising=False)
monkeypatch.delenv("PF1E_PROMPT_PACK_DIR", raising=False)
argv = [*_argv_common(), "--llm-report"]
with pytest.raises(SystemExit) as exc_info:
main(argv)
assert exc_info.value.code == 2
def test_cli_llm_report_requires_prompt_pack_dir(monkeypatch: pytest.MonkeyPatch) -> None:
"""--llm-report fails gracefully when PF1E_PROMPT_PACK_DIR not set."""
monkeypatch.setenv("PF1E_LLM_API_KEY", "test-key")
monkeypatch.delenv("PF1E_PROMPT_PACK_DIR", raising=False)
argv = [*_argv_common(), "--llm-report"]
with pytest.raises(SystemExit) as exc_info:
main(argv)
assert exc_info.value.code == 2
tests/test_format_llm_report.py
+63
View File
@@ -0,0 +1,63 @@
"""Tests for format_llm_report (P5.3 T4)."""
from __future__ import annotations
from pf1e_simulator.report import build_report_context, format_llm_report
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
def _make_encounter() -> EncounterReport:
return EncounterReport(
runs=100,
seed=1,
wins={"players": 65, "monsters": 33},
draws=2,
avg_rounds=7.2,
attrition={
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
},
)
def test_format_llm_report_empty_narrative() -> None:
"""Empty LLM text returns only the factual report."""
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
result = format_llm_report(ctx, "")
assert "=== Rapport d'équilibrage ===" in result
assert "Victoires players : 65.0%" in result
assert "Récit d'équilibrage" not in result
def test_format_llm_report_whitespace_only_narrative() -> None:
"""Whitespace-only LLM text returns only the factual report."""
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
result = format_llm_report(ctx, " \n \t ")
assert "=== Rapport d'équilibrage ===" in result
assert "Récit d'équilibrage" not in result
def test_format_llm_report_with_narrative() -> None:
"""Non-empty LLM text appends narrative section after factual report."""
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
narrative = "Le gobelin a surpris l'orc par sa vitesse."
result = format_llm_report(ctx, narrative)
assert "=== Rapport d'équilibrage ===" in result
assert "Victoires players : 65.0%" in result
assert "=== Récit d'équilibrage (LLM) ===" in result
assert narrative in result
# factual comes before narrative
factual_end = result.index("=== Récit d'équilibrage (LLM) ===")
assert "Victoires players" in result[:factual_end]
def test_format_llm_report_strips_narrative_whitespace() -> None:
"""Leading/trailing whitespace in narrative is stripped."""
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
result = format_llm_report(ctx, " \n Texte narratif. \n ")
assert result.endswith("Texte narratif.")
assert "=== Récit d'équilibrage (LLM) ===\nTexte narratif." in result
tests/test_llm_report_policy.py
+100
View File
@@ -0,0 +1,100 @@
"""Tests for LLMReportPolicy (P5.3 T3)."""
from __future__ import annotations
from pathlib import Path
import pytest
from pf1e_simulator.client import FakeLLMClient
from pf1e_simulator.prompt import PromptPack
from pf1e_simulator.prompt_builder import PromptBuilder
from pf1e_simulator.report import LLMReportPolicy, build_report_context
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
def _make_encounter() -> EncounterReport:
return EncounterReport(
runs=100,
seed=1,
wins={"players": 65, "monsters": 33},
draws=2,
avg_rounds=7.2,
attrition={
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
},
)
def _make_policy(
script: list[str],
*,
max_retries: int = 1,
) -> LLMReportPolicy:
pack = PromptPack.load(FIXTURE_DIR)
builder = PromptBuilder(pack)
expected_version = pack.prompt_version("report")
client = FakeLLMClient(script=script, prompt_version=expected_version) # type: ignore[arg-type]
return LLMReportPolicy(
client=client, builder=builder, prompt_name="report", max_retries=max_retries
)
def test_llm_report_policy_returns_text() -> None:
"""Policy returns the scripted text from FakeLLMClient."""
policy = _make_policy(script=["Rapport narratif de test."])
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
result = policy(ctx)
assert result == "Rapport narratif de test."
assert policy.stats.calls == 1
assert policy.stats.retries == 0
assert policy.stats.fallbacks == 0
def test_llm_report_policy_retry_on_exception() -> None:
"""Policy retries on exception and increments retries counter."""
policy = _make_policy(
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
max_retries=1,
)
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
result = policy(ctx)
assert result == "Rapport après retry."
assert policy.stats.calls == 2
assert policy.stats.retries == 1
assert policy.stats.fallbacks == 0
def test_llm_report_policy_fallback_after_max_retries() -> None:
"""Policy returns empty string and increments fallbacks after max retries."""
policy = _make_policy(
script=[Exception("boom"), Exception("boom2")], # type: ignore[list-item]
max_retries=1,
)
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
result = policy(ctx)
assert result == ""
assert policy.stats.calls == 2
assert policy.stats.retries == 1
assert policy.stats.fallbacks == 1
def test_llm_report_policy_prompt_version_mismatch_raises() -> None:
"""Policy rejects client with mismatched prompt_version."""
pack = PromptPack.load(FIXTURE_DIR)
builder = PromptBuilder(pack)
expected_version = pack.prompt_version("report")
client = FakeLLMClient(script=["x"], prompt_version=expected_version)
client.prompt_version = "wrong-version"
with pytest.raises(ValueError, match="prompt_version"):
LLMReportPolicy(client=client, builder=builder, prompt_name="report")
tests/test_report_budget.py
+162
View File
@@ -0,0 +1,162 @@
"""Tests for ReportBudget and budget exceeded handling (P5.3 T6)."""
from __future__ import annotations
from pathlib import Path
from pf1e_simulator.client import CostBudget, FakeLLMClient
from pf1e_simulator.prompt import PromptPack
from pf1e_simulator.prompt_builder import PromptBuilder
from pf1e_simulator.report import (
LLMReportPolicy,
ReportBudget,
build_report_context,
)
from pf1e_simulator.runner import EncounterReport
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
def _make_policy(
script: list[str],
*,
max_retries: int = 1,
budget: ReportBudget | None = None,
) -> LLMReportPolicy:
pack = PromptPack.load(FIXTURE_DIR)
builder = PromptBuilder(pack)
client = FakeLLMClient(script=script, prompt_version=pack.prompt_version("report")) # type: ignore[arg-type]
return LLMReportPolicy(
client=client,
builder=builder,
prompt_name="report",
max_retries=max_retries,
budget=budget or ReportBudget(),
)
def test_report_budget_default_allows_one_call() -> None:
"""Default budget allows exactly 1 successful call."""
policy = _make_policy(script=["Rapport de test."])
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
result = policy(ctx)
assert result == "Rapport de test."
assert policy.stats.calls == 1
def test_report_budget_exceeded_on_second_call() -> None:
"""Budget exceeded when attempting a second call."""
policy = _make_policy(
script=["First call.", "Second call (should not happen)."],
max_retries=0, # No retries
)
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
# First call succeeds
result1 = policy(ctx)
assert result1 == "First call."
assert policy.stats.calls == 1
# Second call should exceed budget and return empty string (fallback)
result2 = policy(ctx)
assert result2 == ""
assert policy.stats.calls == 2
assert policy.stats.fallbacks == 1
def test_report_budget_for_retries_allows_retry() -> None:
"""Budget adjusted for retries allows the retry to succeed."""
policy = _make_policy(
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
max_retries=1,
)
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
result = policy(ctx)
assert result == "Rapport après retry."
assert policy.stats.calls == 2
assert policy.stats.retries == 1
assert policy.stats.fallbacks == 0
def test_report_budget_for_retries_exceeded() -> None:
"""Budget still exceeded after max retries exhausted."""
policy = _make_policy(
script=[Exception("boom1"), Exception("boom2"), "Should not reach here."], # type: ignore[list-item]
max_retries=1,
)
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
result = policy(ctx)
assert result == ""
assert policy.stats.calls == 2 # initial + 1 retry (budget allows max_retries + 1 = 2)
assert policy.stats.retries == 1
assert policy.stats.fallbacks == 1
def test_report_budget_to_cost_budget() -> None:
"""ReportBudget converts to CostBudget correctly."""
budget = ReportBudget(max_calls=2, max_total_tokens=1000, max_cost_usd=0.01)
cost_budget = budget.to_cost_budget()
assert isinstance(cost_budget, CostBudget)
assert cost_budget.max_calls == 2
assert cost_budget.max_total_tokens == 1000
assert cost_budget.max_cost_usd == 0.01
def test_report_budget_for_retries_adjusts_max_calls() -> None:
"""for_retries adjusts max_calls to max_retries + 1."""
budget = ReportBudget(max_calls=1)
adjusted = budget.for_retries(2)
assert adjusted.max_calls == 3 # 2 retries + 1 initial
assert adjusted.max_total_tokens is None
assert adjusted.max_cost_usd is None
def test_report_budget_for_retries_preserves_other_limits() -> None:
"""for_retries preserves token and cost limits."""
budget = ReportBudget(max_calls=1, max_total_tokens=500, max_cost_usd=0.005)
adjusted = budget.for_retries(3)
assert adjusted.max_calls == 4
assert adjusted.max_total_tokens == 500
assert adjusted.max_cost_usd == 0.005
def test_report_budget_for_retries_preserves_non_default_max_calls() -> None:
"""for_retries preserves non-default max_calls."""
budget = ReportBudget(max_calls=5)
adjusted = budget.for_retries(2)
assert adjusted.max_calls == 5 # unchanged because it wasn't the default 1
tests/test_report_context.py
+139
View File
@@ -0,0 +1,139 @@
"""Tests for ReportContext and build_report_context (P5.3 T1)."""
from __future__ import annotations
from dataclasses import FrozenInstanceError
from pathlib import Path
import pytest
from pf1e_simulator.client import ClientStats
from pf1e_simulator.loaders import load_combatant
from pf1e_simulator.map import load_map
from pf1e_simulator.nl_policy import NLPolicyStats
from pf1e_simulator.objectives import DangerMetrics
from pf1e_simulator.report import ReportContext, build_report_context
from pf1e_simulator.runner import EncounterSpec, Side, run_encounter
from pf1e_simulator.spells import SPELLS_DIR, load_spell_registry
ROOT = Path(__file__).resolve().parents[1]
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
def _c(path: str):
return load_combatant(ROOT / "data" / path)
def _spec() -> EncounterSpec:
"""Fast 1v1 (goblin vs orc) on the sample arena."""
return EncounterSpec(
map=load_map(MAP),
round_cap=20,
sides=(
Side(name="players", combatants=(_c("monsters/goblin.json"),), zone="A"),
Side(name="monsters", combatants=(_c("monsters/orc.json"),), zone="B"),
),
)
def test_build_report_context_minimal() -> None:
"""Minimal context: only EncounterReport, no RunOutcome."""
registry = load_spell_registry(SPELLS_DIR)
spec = _spec()
report = run_encounter(spec, runs=10, seed=1, spell_registry=registry)
ctx = build_report_context(
encounter=report,
spec_name="goblin-vs-orc",
)
assert isinstance(ctx, ReportContext)
assert ctx.encounter is report
assert ctx.spec_name == "goblin-vs-orc"
assert ctx.danger is None
assert ctx.nl_stats is None
assert ctx.client_stats is None
assert ctx.matchup is None
assert ctx.danger_map is None
assert ctx.frontier_value is None
assert ctx.sensitivity is None
def test_build_report_context_with_outcomes() -> None:
"""Context with RunOutcome -> DangerMetrics computed for the party."""
registry = load_spell_registry(SPELLS_DIR)
spec = _spec()
report, outcomes = run_encounter(
spec, runs=100, seed=1, spell_registry=registry, collect_runs=True
)
ctx = build_report_context(
encounter=report,
spec_name="goblin-vs-orc",
outcomes=outcomes,
party="players",
)
assert ctx.danger is not None
assert isinstance(ctx.danger, DangerMetrics)
assert ctx.danger.runs == 100
# p_survives, p_tpk, p_any_death are probabilities in [0, 1]
assert 0.0 <= ctx.danger.p_survives <= 1.0
assert 0.0 <= ctx.danger.p_tpk <= 1.0
assert 0.0 <= ctx.danger.p_any_death <= 1.0
assert ctx.danger.expected_hp_lost >= 0.0
assert ctx.danger.expected_rounds > 0.0
assert ctx.danger.worst_decile_party_hp >= 0.0
def test_build_report_context_with_all_optional_fields() -> None:
"""Context with all optional fields populated."""
registry = load_spell_registry(SPELLS_DIR)
spec = _spec()
report, outcomes = run_encounter(
spec, runs=100, seed=1, spell_registry=registry, collect_runs=True
)
nl_stats = NLPolicyStats(calls=5, repairs_attempted=1, repairs_succeeded=1, fallbacks=0)
client_stats = ClientStats(
calls=5,
prompt_tokens=1000,
completion_tokens=200,
cost_usd=0.001,
cache_hits=0,
cache_misses=5,
)
ctx = build_report_context(
encounter=report,
spec_name="goblin-vs-orc",
outcomes=outcomes,
party="players",
nl_stats=nl_stats,
client_stats=client_stats,
)
assert ctx.nl_stats is nl_stats
assert ctx.client_stats is client_stats
assert ctx.nl_stats.calls == 5 # type: ignore[reportOptionalMemberAccess]
assert ctx.nl_stats.repairs_attempted == 1 # type: ignore[reportOptionalMemberAccess]
assert ctx.nl_stats.repairs_succeeded == 1 # type: ignore[reportOptionalMemberAccess]
assert ctx.nl_stats.fallbacks == 0 # type: ignore[reportOptionalMemberAccess]
assert ctx.client_stats.prompt_tokens == 1000 # type: ignore[reportOptionalMemberAccess]
assert ctx.client_stats.cost_usd == 0.001 # type: ignore[reportOptionalMemberAccess]
def test_report_context_immutable() -> None:
"""ReportContext is a frozen dataclass (immutable)."""
registry = load_spell_registry(SPELLS_DIR)
spec = _spec()
report = run_encounter(spec, runs=5, seed=1, spell_registry=registry)
ctx = build_report_context(
encounter=report,
spec_name="test",
)
# Frozen dataclass -> assignment raises FrozenInstanceError
with pytest.raises(FrozenInstanceError):
ctx.encounter = None # type: ignore[misc]
tests/test_report_prompt_pack.py
+67
View File
@@ -0,0 +1,67 @@
"""Tests for report prompt pack (P5.3 T2)."""
from __future__ import annotations
from pathlib import Path
from pf1e_simulator.prompt import PromptPack
from pf1e_simulator.report import build_report_context, build_report_prompt
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
def test_report_prompt_pack_version_stable() -> None:
"""Prompt pack version hash is stable for the fixture."""
pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
pack = PromptPack.load(pack_dir)
report_version = pack.prompt_version("report")
assert report_version.startswith("0.1.0-test:report:")
assert len(report_version.split(":")[2]) == 12 # hash12
def test_report_prompt_renders() -> None:
"""build_report_prompt renders the context correctly."""
pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
pack = PromptPack.load(pack_dir)
template = pack.get("report")
encounter = EncounterReport(
runs=100,
seed=1,
wins={"players": 65, "monsters": 33},
draws=2,
avg_rounds=7.2,
attrition={
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
},
)
ctx = build_report_context(
encounter=encounter,
spec_name="test",
)
prompt = build_report_prompt(ctx, template)
# Should contain the system prompt text
assert "Vous êtes le narrateur" in prompt
assert "Bilan d'équilibrage" in prompt
assert "Danger et attrition" in prompt
# Should contain the rendered encounter data
assert "Victoires players : 65.0%" in prompt or "Victoires players : 65" in prompt
assert "goblin" in prompt
assert "orc" in prompt
# Should contain attrition
assert "Attrition moyenne par combatant" in prompt
assert "touches" in prompt
def test_report_prompt_version_different_from_nl_default() -> None:
"""report prompt_version differs from nl_default."""
pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
pack = PromptPack.load(pack_dir)
report_version = pack.prompt_version("report")
nl_version = pack.prompt_version("nl_default")
assert report_version != nl_version