feat(phase2): llm report writer (P5.3)

- New module src/pf1e_simulator/report.py with ReportContext, build_report_context,
  LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt
- Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator)
- CLI integration: --llm-report + --output flags, mutually exclusive with --transcript
- Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls
- Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy,
  test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
This commit is contained in:
2026-09-26 10:40:58 +02:00
parent c454a67b84
commit 98985ee6fc
9 changed files with 1024 additions and 12 deletions
+162
View File
@@ -0,0 +1,162 @@
"""Tests for ReportBudget and budget exceeded handling (P5.3 T6)."""
from __future__ import annotations
from pathlib import Path
from pf1e_simulator.client import CostBudget, FakeLLMClient
from pf1e_simulator.prompt import PromptPack
from pf1e_simulator.prompt_builder import PromptBuilder
from pf1e_simulator.report import (
LLMReportPolicy,
ReportBudget,
build_report_context,
)
from pf1e_simulator.runner import EncounterReport
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
def _make_policy(
script: list[str],
*,
max_retries: int = 1,
budget: ReportBudget | None = None,
) -> LLMReportPolicy:
pack = PromptPack.load(FIXTURE_DIR)
builder = PromptBuilder(pack)
client = FakeLLMClient(script=script, prompt_version=pack.prompt_version("report")) # type: ignore[arg-type]
return LLMReportPolicy(
client=client,
builder=builder,
prompt_name="report",
max_retries=max_retries,
budget=budget or ReportBudget(),
)
def test_report_budget_default_allows_one_call() -> None:
"""Default budget allows exactly 1 successful call."""
policy = _make_policy(script=["Rapport de test."])
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
result = policy(ctx)
assert result == "Rapport de test."
assert policy.stats.calls == 1
def test_report_budget_exceeded_on_second_call() -> None:
"""Budget exceeded when attempting a second call."""
policy = _make_policy(
script=["First call.", "Second call (should not happen)."],
max_retries=0, # No retries
)
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
# First call succeeds
result1 = policy(ctx)
assert result1 == "First call."
assert policy.stats.calls == 1
# Second call should exceed budget and return empty string (fallback)
result2 = policy(ctx)
assert result2 == ""
assert policy.stats.calls == 2
assert policy.stats.fallbacks == 1
def test_report_budget_for_retries_allows_retry() -> None:
"""Budget adjusted for retries allows the retry to succeed."""
policy = _make_policy(
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
max_retries=1,
)
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
result = policy(ctx)
assert result == "Rapport après retry."
assert policy.stats.calls == 2
assert policy.stats.retries == 1
assert policy.stats.fallbacks == 0
def test_report_budget_for_retries_exceeded() -> None:
"""Budget still exceeded after max retries exhausted."""
policy = _make_policy(
script=[Exception("boom1"), Exception("boom2"), "Should not reach here."], # type: ignore[list-item]
max_retries=1,
)
ctx = build_report_context(
encounter=EncounterReport(
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
attrition={}
),
spec_name="test",
)
result = policy(ctx)
assert result == ""
assert policy.stats.calls == 2 # initial + 1 retry (budget allows max_retries + 1 = 2)
assert policy.stats.retries == 1
assert policy.stats.fallbacks == 1
def test_report_budget_to_cost_budget() -> None:
"""ReportBudget converts to CostBudget correctly."""
budget = ReportBudget(max_calls=2, max_total_tokens=1000, max_cost_usd=0.01)
cost_budget = budget.to_cost_budget()
assert isinstance(cost_budget, CostBudget)
assert cost_budget.max_calls == 2
assert cost_budget.max_total_tokens == 1000
assert cost_budget.max_cost_usd == 0.01
def test_report_budget_for_retries_adjusts_max_calls() -> None:
"""for_retries adjusts max_calls to max_retries + 1."""
budget = ReportBudget(max_calls=1)
adjusted = budget.for_retries(2)
assert adjusted.max_calls == 3 # 2 retries + 1 initial
assert adjusted.max_total_tokens is None
assert adjusted.max_cost_usd is None
def test_report_budget_for_retries_preserves_other_limits() -> None:
"""for_retries preserves token and cost limits."""
budget = ReportBudget(max_calls=1, max_total_tokens=500, max_cost_usd=0.005)
adjusted = budget.for_retries(3)
assert adjusted.max_calls == 4
assert adjusted.max_total_tokens == 500
assert adjusted.max_cost_usd == 0.005
def test_report_budget_for_retries_preserves_non_default_max_calls() -> None:
"""for_retries preserves non-default max_calls."""
budget = ReportBudget(max_calls=5)
adjusted = budget.for_retries(2)
assert adjusted.max_calls == 5 # unchanged because it wasn't the default 1