ctan
98985ee6fc
- New module src/pf1e_simulator/report.py with ReportContext, build_report_context, LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt - Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator) - CLI integration: --llm-report + --output flags, mutually exclusive with --transcript - Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls - Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy, test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
101 lines
3.3 KiB
Python
101 lines
3.3 KiB
Python
"""Tests for LLMReportPolicy (P5.3 T3)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from pf1e_simulator.client import FakeLLMClient
|
|
from pf1e_simulator.prompt import PromptPack
|
|
from pf1e_simulator.prompt_builder import PromptBuilder
|
|
from pf1e_simulator.report import LLMReportPolicy, build_report_context
|
|
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
|
|
|
|
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
|
|
|
|
|
def _make_encounter() -> EncounterReport:
|
|
return EncounterReport(
|
|
runs=100,
|
|
seed=1,
|
|
wins={"players": 65, "monsters": 33},
|
|
draws=2,
|
|
avg_rounds=7.2,
|
|
attrition={
|
|
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
|
|
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
|
|
},
|
|
)
|
|
|
|
|
|
def _make_policy(
|
|
script: list[str],
|
|
*,
|
|
max_retries: int = 1,
|
|
) -> LLMReportPolicy:
|
|
pack = PromptPack.load(FIXTURE_DIR)
|
|
builder = PromptBuilder(pack)
|
|
expected_version = pack.prompt_version("report")
|
|
client = FakeLLMClient(script=script, prompt_version=expected_version) # type: ignore[arg-type]
|
|
return LLMReportPolicy(
|
|
client=client, builder=builder, prompt_name="report", max_retries=max_retries
|
|
)
|
|
|
|
|
|
def test_llm_report_policy_returns_text() -> None:
|
|
"""Policy returns the scripted text from FakeLLMClient."""
|
|
policy = _make_policy(script=["Rapport narratif de test."])
|
|
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
|
|
|
result = policy(ctx)
|
|
|
|
assert result == "Rapport narratif de test."
|
|
assert policy.stats.calls == 1
|
|
assert policy.stats.retries == 0
|
|
assert policy.stats.fallbacks == 0
|
|
|
|
|
|
def test_llm_report_policy_retry_on_exception() -> None:
|
|
"""Policy retries on exception and increments retries counter."""
|
|
policy = _make_policy(
|
|
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
|
|
max_retries=1,
|
|
)
|
|
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
|
|
|
result = policy(ctx)
|
|
|
|
assert result == "Rapport après retry."
|
|
assert policy.stats.calls == 2
|
|
assert policy.stats.retries == 1
|
|
assert policy.stats.fallbacks == 0
|
|
|
|
|
|
def test_llm_report_policy_fallback_after_max_retries() -> None:
|
|
"""Policy returns empty string and increments fallbacks after max retries."""
|
|
policy = _make_policy(
|
|
script=[Exception("boom"), Exception("boom2")], # type: ignore[list-item]
|
|
max_retries=1,
|
|
)
|
|
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
|
|
|
result = policy(ctx)
|
|
|
|
assert result == ""
|
|
assert policy.stats.calls == 2
|
|
assert policy.stats.retries == 1
|
|
assert policy.stats.fallbacks == 1
|
|
|
|
|
|
def test_llm_report_policy_prompt_version_mismatch_raises() -> None:
|
|
"""Policy rejects client with mismatched prompt_version."""
|
|
pack = PromptPack.load(FIXTURE_DIR)
|
|
builder = PromptBuilder(pack)
|
|
expected_version = pack.prompt_version("report")
|
|
client = FakeLLMClient(script=["x"], prompt_version=expected_version)
|
|
client.prompt_version = "wrong-version"
|
|
|
|
with pytest.raises(ValueError, match="prompt_version"):
|
|
LLMReportPolicy(client=client, builder=builder, prompt_name="report")
|