feat(phase2): llm report writer (P5.3)
- New module src/pf1e_simulator/report.py with ReportContext, build_report_context, LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt - Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator) - CLI integration: --llm-report + --output flags, mutually exclusive with --transcript - Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls - Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy, test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
"""Tests for ReportBudget and budget exceeded handling (P5.3 T6)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pf1e_simulator.client import CostBudget, FakeLLMClient
|
||||
from pf1e_simulator.prompt import PromptPack
|
||||
from pf1e_simulator.prompt_builder import PromptBuilder
|
||||
from pf1e_simulator.report import (
|
||||
LLMReportPolicy,
|
||||
ReportBudget,
|
||||
build_report_context,
|
||||
)
|
||||
from pf1e_simulator.runner import EncounterReport
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
|
||||
|
||||
def _make_policy(
|
||||
script: list[str],
|
||||
*,
|
||||
max_retries: int = 1,
|
||||
budget: ReportBudget | None = None,
|
||||
) -> LLMReportPolicy:
|
||||
pack = PromptPack.load(FIXTURE_DIR)
|
||||
builder = PromptBuilder(pack)
|
||||
client = FakeLLMClient(script=script, prompt_version=pack.prompt_version("report")) # type: ignore[arg-type]
|
||||
return LLMReportPolicy(
|
||||
client=client,
|
||||
builder=builder,
|
||||
prompt_name="report",
|
||||
max_retries=max_retries,
|
||||
budget=budget or ReportBudget(),
|
||||
)
|
||||
|
||||
|
||||
def test_report_budget_default_allows_one_call() -> None:
|
||||
"""Default budget allows exactly 1 successful call."""
|
||||
policy = _make_policy(script=["Rapport de test."])
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport de test."
|
||||
assert policy.stats.calls == 1
|
||||
|
||||
|
||||
def test_report_budget_exceeded_on_second_call() -> None:
|
||||
"""Budget exceeded when attempting a second call."""
|
||||
policy = _make_policy(
|
||||
script=["First call.", "Second call (should not happen)."],
|
||||
max_retries=0, # No retries
|
||||
)
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
# First call succeeds
|
||||
result1 = policy(ctx)
|
||||
assert result1 == "First call."
|
||||
assert policy.stats.calls == 1
|
||||
|
||||
# Second call should exceed budget and return empty string (fallback)
|
||||
result2 = policy(ctx)
|
||||
assert result2 == ""
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.fallbacks == 1
|
||||
|
||||
|
||||
def test_report_budget_for_retries_allows_retry() -> None:
|
||||
"""Budget adjusted for retries allows the retry to succeed."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport après retry."
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 0
|
||||
|
||||
|
||||
def test_report_budget_for_retries_exceeded() -> None:
|
||||
"""Budget still exceeded after max retries exhausted."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom1"), Exception("boom2"), "Should not reach here."], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == ""
|
||||
assert policy.stats.calls == 2 # initial + 1 retry (budget allows max_retries + 1 = 2)
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 1
|
||||
|
||||
|
||||
def test_report_budget_to_cost_budget() -> None:
|
||||
"""ReportBudget converts to CostBudget correctly."""
|
||||
budget = ReportBudget(max_calls=2, max_total_tokens=1000, max_cost_usd=0.01)
|
||||
cost_budget = budget.to_cost_budget()
|
||||
|
||||
assert isinstance(cost_budget, CostBudget)
|
||||
assert cost_budget.max_calls == 2
|
||||
assert cost_budget.max_total_tokens == 1000
|
||||
assert cost_budget.max_cost_usd == 0.01
|
||||
|
||||
|
||||
def test_report_budget_for_retries_adjusts_max_calls() -> None:
|
||||
"""for_retries adjusts max_calls to max_retries + 1."""
|
||||
budget = ReportBudget(max_calls=1)
|
||||
adjusted = budget.for_retries(2)
|
||||
|
||||
assert adjusted.max_calls == 3 # 2 retries + 1 initial
|
||||
assert adjusted.max_total_tokens is None
|
||||
assert adjusted.max_cost_usd is None
|
||||
|
||||
|
||||
def test_report_budget_for_retries_preserves_other_limits() -> None:
|
||||
"""for_retries preserves token and cost limits."""
|
||||
budget = ReportBudget(max_calls=1, max_total_tokens=500, max_cost_usd=0.005)
|
||||
adjusted = budget.for_retries(3)
|
||||
|
||||
assert adjusted.max_calls == 4
|
||||
assert adjusted.max_total_tokens == 500
|
||||
assert adjusted.max_cost_usd == 0.005
|
||||
|
||||
|
||||
def test_report_budget_for_retries_preserves_non_default_max_calls() -> None:
|
||||
"""for_retries preserves non-default max_calls."""
|
||||
budget = ReportBudget(max_calls=5)
|
||||
adjusted = budget.for_retries(2)
|
||||
|
||||
assert adjusted.max_calls == 5 # unchanged because it wasn't the default 1
|
||||
|
||||
Reference in New Issue
Block a user