ctan
98985ee6fc
- New module src/pf1e_simulator/report.py with ReportContext, build_report_context, LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt - Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator) - CLI integration: --llm-report + --output flags, mutually exclusive with --transcript - Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls - Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy, test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
140 lines
4.6 KiB
Python
140 lines
4.6 KiB
Python
"""Tests for ReportContext and build_report_context (P5.3 T1)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import FrozenInstanceError
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from pf1e_simulator.client import ClientStats
|
|
from pf1e_simulator.loaders import load_combatant
|
|
from pf1e_simulator.map import load_map
|
|
from pf1e_simulator.nl_policy import NLPolicyStats
|
|
from pf1e_simulator.objectives import DangerMetrics
|
|
from pf1e_simulator.report import ReportContext, build_report_context
|
|
from pf1e_simulator.runner import EncounterSpec, Side, run_encounter
|
|
from pf1e_simulator.spells import SPELLS_DIR, load_spell_registry
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
|
|
|
|
|
def _c(path: str):
|
|
return load_combatant(ROOT / "data" / path)
|
|
|
|
|
|
def _spec() -> EncounterSpec:
|
|
"""Fast 1v1 (goblin vs orc) on the sample arena."""
|
|
return EncounterSpec(
|
|
map=load_map(MAP),
|
|
round_cap=20,
|
|
sides=(
|
|
Side(name="players", combatants=(_c("monsters/goblin.json"),), zone="A"),
|
|
Side(name="monsters", combatants=(_c("monsters/orc.json"),), zone="B"),
|
|
),
|
|
)
|
|
|
|
|
|
def test_build_report_context_minimal() -> None:
|
|
"""Minimal context: only EncounterReport, no RunOutcome."""
|
|
registry = load_spell_registry(SPELLS_DIR)
|
|
spec = _spec()
|
|
report = run_encounter(spec, runs=10, seed=1, spell_registry=registry)
|
|
|
|
ctx = build_report_context(
|
|
encounter=report,
|
|
spec_name="goblin-vs-orc",
|
|
)
|
|
|
|
assert isinstance(ctx, ReportContext)
|
|
assert ctx.encounter is report
|
|
assert ctx.spec_name == "goblin-vs-orc"
|
|
assert ctx.danger is None
|
|
assert ctx.nl_stats is None
|
|
assert ctx.client_stats is None
|
|
assert ctx.matchup is None
|
|
assert ctx.danger_map is None
|
|
assert ctx.frontier_value is None
|
|
assert ctx.sensitivity is None
|
|
|
|
|
|
def test_build_report_context_with_outcomes() -> None:
|
|
"""Context with RunOutcome -> DangerMetrics computed for the party."""
|
|
registry = load_spell_registry(SPELLS_DIR)
|
|
spec = _spec()
|
|
report, outcomes = run_encounter(
|
|
spec, runs=100, seed=1, spell_registry=registry, collect_runs=True
|
|
)
|
|
|
|
ctx = build_report_context(
|
|
encounter=report,
|
|
spec_name="goblin-vs-orc",
|
|
outcomes=outcomes,
|
|
party="players",
|
|
)
|
|
|
|
assert ctx.danger is not None
|
|
assert isinstance(ctx.danger, DangerMetrics)
|
|
assert ctx.danger.runs == 100
|
|
# p_survives, p_tpk, p_any_death are probabilities in [0, 1]
|
|
assert 0.0 <= ctx.danger.p_survives <= 1.0
|
|
assert 0.0 <= ctx.danger.p_tpk <= 1.0
|
|
assert 0.0 <= ctx.danger.p_any_death <= 1.0
|
|
assert ctx.danger.expected_hp_lost >= 0.0
|
|
assert ctx.danger.expected_rounds > 0.0
|
|
assert ctx.danger.worst_decile_party_hp >= 0.0
|
|
|
|
|
|
def test_build_report_context_with_all_optional_fields() -> None:
|
|
"""Context with all optional fields populated."""
|
|
registry = load_spell_registry(SPELLS_DIR)
|
|
spec = _spec()
|
|
report, outcomes = run_encounter(
|
|
spec, runs=100, seed=1, spell_registry=registry, collect_runs=True
|
|
)
|
|
|
|
nl_stats = NLPolicyStats(calls=5, repairs_attempted=1, repairs_succeeded=1, fallbacks=0)
|
|
client_stats = ClientStats(
|
|
calls=5,
|
|
prompt_tokens=1000,
|
|
completion_tokens=200,
|
|
cost_usd=0.001,
|
|
cache_hits=0,
|
|
cache_misses=5,
|
|
)
|
|
|
|
ctx = build_report_context(
|
|
encounter=report,
|
|
spec_name="goblin-vs-orc",
|
|
outcomes=outcomes,
|
|
party="players",
|
|
nl_stats=nl_stats,
|
|
client_stats=client_stats,
|
|
)
|
|
|
|
assert ctx.nl_stats is nl_stats
|
|
assert ctx.client_stats is client_stats
|
|
assert ctx.nl_stats.calls == 5 # type: ignore[reportOptionalMemberAccess]
|
|
assert ctx.nl_stats.repairs_attempted == 1 # type: ignore[reportOptionalMemberAccess]
|
|
assert ctx.nl_stats.repairs_succeeded == 1 # type: ignore[reportOptionalMemberAccess]
|
|
assert ctx.nl_stats.fallbacks == 0 # type: ignore[reportOptionalMemberAccess]
|
|
assert ctx.client_stats.prompt_tokens == 1000 # type: ignore[reportOptionalMemberAccess]
|
|
assert ctx.client_stats.cost_usd == 0.001 # type: ignore[reportOptionalMemberAccess]
|
|
|
|
|
|
def test_report_context_immutable() -> None:
|
|
"""ReportContext is a frozen dataclass (immutable)."""
|
|
registry = load_spell_registry(SPELLS_DIR)
|
|
spec = _spec()
|
|
report = run_encounter(spec, runs=5, seed=1, spell_registry=registry)
|
|
|
|
ctx = build_report_context(
|
|
encounter=report,
|
|
spec_name="test",
|
|
)
|
|
|
|
# Frozen dataclass -> assignment raises FrozenInstanceError
|
|
with pytest.raises(FrozenInstanceError):
|
|
ctx.encounter = None # type: ignore[misc]
|