feat(phase2): llm report writer (P5.3)
- New module src/pf1e_simulator/report.py with ReportContext, build_report_context, LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt - Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator) - CLI integration: --llm-report + --output flags, mutually exclusive with --transcript - Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls - Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy, test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
"""Tests for LLMReportPolicy (P5.3 T3)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.client import FakeLLMClient
|
||||
from pf1e_simulator.prompt import PromptPack
|
||||
from pf1e_simulator.prompt_builder import PromptBuilder
|
||||
from pf1e_simulator.report import LLMReportPolicy, build_report_context
|
||||
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
|
||||
|
||||
def _make_encounter() -> EncounterReport:
|
||||
return EncounterReport(
|
||||
runs=100,
|
||||
seed=1,
|
||||
wins={"players": 65, "monsters": 33},
|
||||
draws=2,
|
||||
avg_rounds=7.2,
|
||||
attrition={
|
||||
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
|
||||
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_policy(
|
||||
script: list[str],
|
||||
*,
|
||||
max_retries: int = 1,
|
||||
) -> LLMReportPolicy:
|
||||
pack = PromptPack.load(FIXTURE_DIR)
|
||||
builder = PromptBuilder(pack)
|
||||
expected_version = pack.prompt_version("report")
|
||||
client = FakeLLMClient(script=script, prompt_version=expected_version) # type: ignore[arg-type]
|
||||
return LLMReportPolicy(
|
||||
client=client, builder=builder, prompt_name="report", max_retries=max_retries
|
||||
)
|
||||
|
||||
|
||||
def test_llm_report_policy_returns_text() -> None:
|
||||
"""Policy returns the scripted text from FakeLLMClient."""
|
||||
policy = _make_policy(script=["Rapport narratif de test."])
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport narratif de test."
|
||||
assert policy.stats.calls == 1
|
||||
assert policy.stats.retries == 0
|
||||
assert policy.stats.fallbacks == 0
|
||||
|
||||
|
||||
def test_llm_report_policy_retry_on_exception() -> None:
|
||||
"""Policy retries on exception and increments retries counter."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport après retry."
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 0
|
||||
|
||||
|
||||
def test_llm_report_policy_fallback_after_max_retries() -> None:
|
||||
"""Policy returns empty string and increments fallbacks after max retries."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom"), Exception("boom2")], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == ""
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 1
|
||||
|
||||
|
||||
def test_llm_report_policy_prompt_version_mismatch_raises() -> None:
|
||||
"""Policy rejects client with mismatched prompt_version."""
|
||||
pack = PromptPack.load(FIXTURE_DIR)
|
||||
builder = PromptBuilder(pack)
|
||||
expected_version = pack.prompt_version("report")
|
||||
client = FakeLLMClient(script=["x"], prompt_version=expected_version)
|
||||
client.prompt_version = "wrong-version"
|
||||
|
||||
with pytest.raises(ValueError, match="prompt_version"):
|
||||
LLMReportPolicy(client=client, builder=builder, prompt_name="report")
|
||||
Reference in New Issue
Block a user