feat(phase2): llm report writer (P5.3)
- New module src/pf1e_simulator/report.py with ReportContext, build_report_context, LLMReportPolicy, ReportBudget, format_llm_report, build_report_prompt - Prompt pack fixture: tests/fixtures/prompt_pack_fs/report.md (FR narrator) - CLI integration: --llm-report + --output flags, mutually exclusive with --transcript - Budget enforcement: ReportBudget integrates with CostBudget, allows max_retries+1 calls - Tests: test_report_context, test_report_prompt_pack, test_llm_report_policy, test_format_llm_report, test_cli_llm_report, test_report_budget (27 new tests)
This commit is contained in:
+119
-12
@@ -19,11 +19,24 @@ if TYPE_CHECKING:
|
||||
|
||||
from pf1e_simulator.runner import EncounterReport, EncounterSpec
|
||||
|
||||
from pf1e_simulator.config import ENV_PROMPT_PACK_DIR, env_path
|
||||
from pf1e_simulator.loaders import CombatantLoadError, SheetLoadError, load_combatant
|
||||
from pf1e_simulator.map import MapValidationError, load_map
|
||||
from pf1e_simulator.runner import EncounterSpec, Side, run_encounter
|
||||
from pf1e_simulator.prompt import PromptPack
|
||||
from pf1e_simulator.prompt_builder import PromptBuilder
|
||||
from pf1e_simulator.report import (
|
||||
LLMReportPolicy,
|
||||
build_report_context,
|
||||
format_llm_report,
|
||||
)
|
||||
from pf1e_simulator.runner import EncounterSpec, RunOutcome, Side, run_encounter
|
||||
from pf1e_simulator.spells import SPELLS_DIR, load_spell_registry
|
||||
|
||||
try:
|
||||
from pf1e_simulator.provider import OpenAILLMClient
|
||||
except ImportError: # pragma: no cover - optional dependency
|
||||
OpenAILLMClient = None # type: ignore[assignment,misc]
|
||||
|
||||
|
||||
class CliError(Exception):
|
||||
"""User-facing CLI error (bad arguments or unusable files)."""
|
||||
@@ -129,7 +142,7 @@ def _build_spec(args: argparse.Namespace) -> EncounterSpec:
|
||||
raise CliError(msg)
|
||||
unknown = sorted(provided_names - set(map_spec.deployment))
|
||||
if unknown:
|
||||
msg = f"--side names not in map deployment: {', '.join(unknown)}"
|
||||
msg = f"--side names not in map deployment: {', '.join(sorted(unknown))}"
|
||||
raise CliError(msg)
|
||||
sides: list[Side] = []
|
||||
for raw in provided:
|
||||
@@ -172,6 +185,56 @@ def format_report(report: EncounterReport, spec: EncounterSpec) -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _run_llm_report(
|
||||
spec: EncounterSpec,
|
||||
args: argparse.Namespace,
|
||||
registry: dict[str, object] | None,
|
||||
) -> str:
|
||||
"""Generate the LLM-powered report."""
|
||||
if OpenAILLMClient is None:
|
||||
print("pf1e-sim: LLM provider not available", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
pack_dir = env_path(ENV_PROMPT_PACK_DIR)
|
||||
if pack_dir is None:
|
||||
print(f"pf1e-sim: {ENV_PROMPT_PACK_DIR} environment variable not set", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
pack = PromptPack.load(pack_dir)
|
||||
prompt_version = pack.prompt_version("report")
|
||||
builder = PromptBuilder(pack)
|
||||
|
||||
client = OpenAILLMClient(
|
||||
prompt_version=prompt_version,
|
||||
model=args.llm_model,
|
||||
api_key=args.llm_api_key,
|
||||
base_url=args.llm_base_url,
|
||||
)
|
||||
|
||||
report: EncounterReport
|
||||
outcomes: tuple[RunOutcome, ...]
|
||||
report, outcomes = run_encounter( # type: ignore[misc]
|
||||
spec,
|
||||
runs=args.runs,
|
||||
seed=args.seed,
|
||||
spell_registry=registry, # type: ignore[arg-type]
|
||||
capture_first_transcript=args.transcript,
|
||||
monsters_die_at_zero=args.simple_monster_death,
|
||||
collect_runs=True,
|
||||
)
|
||||
|
||||
ctx = build_report_context(
|
||||
encounter=report, # type: ignore[arg-type]
|
||||
spec_name=spec.map.name,
|
||||
outcomes=outcomes, # type: ignore[arg-type]
|
||||
party="players",
|
||||
)
|
||||
|
||||
policy = LLMReportPolicy(client=client, builder=builder, prompt_name="report")
|
||||
narrative = policy(ctx)
|
||||
return format_llm_report(ctx, narrative)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""CLI entry point; returns the process exit code."""
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -207,9 +270,39 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"(player characters are never affected)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--llm-report",
|
||||
action="store_true",
|
||||
help="generate an LLM-powered French narrative report (requires PF1E_LLM_API_KEY)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
help="write the full report (factual + LLM narrative) to this file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--llm-model",
|
||||
type=str,
|
||||
help="override the LLM model (default: PF1E_LLM_MODEL env or gpt-4o-mini)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--llm-base-url",
|
||||
type=str,
|
||||
help="override the LLM base URL (default: PF1E_LLM_BASE_URL env or https://api.openai.com/v1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--llm-api-key",
|
||||
type=str,
|
||||
help="override the LLM API key (default: PF1E_LLM_API_KEY env)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
if not args.side:
|
||||
parser.error("at least one --side is required")
|
||||
if args.llm_report and args.transcript:
|
||||
parser.error("--llm-report and --transcript are mutually exclusive")
|
||||
if args.output and not args.llm_report:
|
||||
parser.error("--output requires --llm-report")
|
||||
|
||||
try:
|
||||
spec = _build_spec(args)
|
||||
except CliError as exc:
|
||||
@@ -218,22 +311,36 @@ def main(argv: list[str] | None = None) -> int:
|
||||
except (MapValidationError, CombatantLoadError, SheetLoadError, OSError) as exc:
|
||||
print(f"pf1e-sim: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
registry = load_spell_registry(SPELLS_DIR)
|
||||
report = run_encounter(
|
||||
spec,
|
||||
runs=args.runs,
|
||||
seed=args.seed,
|
||||
spell_registry=registry,
|
||||
capture_first_transcript=args.transcript,
|
||||
monsters_die_at_zero=args.simple_monster_death,
|
||||
)
|
||||
|
||||
report = None
|
||||
if args.llm_report:
|
||||
full_report = _run_llm_report(spec, args, registry) # type: ignore[arg-type]
|
||||
else:
|
||||
report = run_encounter(
|
||||
spec,
|
||||
runs=args.runs,
|
||||
seed=args.seed,
|
||||
spell_registry=registry,
|
||||
capture_first_transcript=args.transcript,
|
||||
monsters_die_at_zero=args.simple_monster_death,
|
||||
)
|
||||
full_report = format_report(report, spec)
|
||||
|
||||
except ValueError as exc:
|
||||
print(f"pf1e-sim: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
if args.transcript and report.transcript is not None:
|
||||
|
||||
if args.transcript and report is not None and report.transcript is not None:
|
||||
print("=== Déroulé du premier combat ===")
|
||||
print(render_transcript_table(report.transcript))
|
||||
print()
|
||||
print(format_report(report, spec))
|
||||
|
||||
print(full_report)
|
||||
|
||||
if args.output:
|
||||
args.output.write_text(full_report, encoding="utf-8")
|
||||
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
"""LLM report writer (Phase 3) — aggregator + French narrative generator.
|
||||
|
||||
This module provides the data structures and pure functions to build a
|
||||
report context from Monte Carlo outcomes and LLM observability counters,
|
||||
then format the final French report combining the factual balance report
|
||||
with an LLM-generated narrative section.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pf1e_simulator.client import BudgetExceededError, CostBudget
|
||||
from pf1e_simulator.objectives import danger_metrics
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
from pf1e_simulator.client import ClientStats, LLMClient
|
||||
from pf1e_simulator.nl_policy import NLPolicyStats
|
||||
from pf1e_simulator.objectives import DangerMetrics
|
||||
from pf1e_simulator.prompt_builder import PromptBuilder
|
||||
from pf1e_simulator.robustness import (
|
||||
DangerCell,
|
||||
MatchupBenchmark,
|
||||
SensitivityEntry,
|
||||
)
|
||||
from pf1e_simulator.runner import EncounterReport, RunOutcome
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReportContext:
|
||||
"""Complete context for the LLM report writer.
|
||||
|
||||
Fields marked as optional are filled when the corresponding analysis
|
||||
was executed (e.g. `collect_runs=True` for DangerMetrics, robustness
|
||||
calls for MatchupBenchmark/SensitivityEntry/DangerCell).
|
||||
"""
|
||||
|
||||
encounter: EncounterReport
|
||||
spec_name: str
|
||||
|
||||
danger: DangerMetrics | None = None
|
||||
|
||||
nl_stats: NLPolicyStats | None = None
|
||||
client_stats: ClientStats | None = None
|
||||
|
||||
matchup: MatchupBenchmark | None = None
|
||||
danger_map: tuple[DangerCell, ...] | None = None
|
||||
frontier_value: float | None = None
|
||||
sensitivity: tuple[SensitivityEntry, ...] | None = None
|
||||
|
||||
|
||||
def build_report_context( # noqa: PLR0913 - intentional builder with many optional params
|
||||
*,
|
||||
encounter: EncounterReport,
|
||||
spec_name: str,
|
||||
outcomes: Sequence[RunOutcome] | None = None,
|
||||
party: str = "players",
|
||||
nl_stats: NLPolicyStats | None = None,
|
||||
client_stats: ClientStats | None = None,
|
||||
matchup: MatchupBenchmark | None = None,
|
||||
danger_map: tuple[DangerCell, ...] | None = None,
|
||||
frontier_value: float | None = None,
|
||||
sensitivity: tuple[SensitivityEntry, ...] | None = None,
|
||||
) -> ReportContext:
|
||||
"""Build a `ReportContext` from raw Monte Carlo outputs."""
|
||||
danger: DangerMetrics | None = None
|
||||
if outcomes is not None:
|
||||
danger = danger_metrics(outcomes, party)
|
||||
|
||||
return ReportContext(
|
||||
encounter=encounter,
|
||||
spec_name=spec_name,
|
||||
danger=danger,
|
||||
nl_stats=nl_stats,
|
||||
client_stats=client_stats,
|
||||
matchup=matchup,
|
||||
danger_map=danger_map,
|
||||
frontier_value=frontier_value,
|
||||
sensitivity=sensitivity,
|
||||
)
|
||||
|
||||
|
||||
def _render_encounter(ctx: ReportContext) -> str:
|
||||
"""Render the EncounterReport in the same format as cli.format_report."""
|
||||
r = ctx.encounter
|
||||
lines = ["=== Rapport d'équilibrage ===", f"Carte : {ctx.spec_name}"]
|
||||
lines.extend(f"{side_name} : {r.wins.get(side_name, 0)} combats" for side_name in r.wins)
|
||||
lines.append(f"{r.runs} combats simulés (seed {r.seed}, plafond ? rounds)")
|
||||
lines.append("")
|
||||
for side_name in r.wins:
|
||||
wins = r.wins.get(side_name, 0)
|
||||
rate = r.win_rate(side_name)
|
||||
low, high = r.win_rate_band(side_name)
|
||||
lines.append(
|
||||
f"Victoires {side_name} : {rate:.1%} ({wins}) — "
|
||||
f"bande 3σ : [{low:.1%}, {high:.1%}]"
|
||||
)
|
||||
lines.append(f"Nuls : {r.draws / r.runs:.1%} ({r.draws})")
|
||||
lines.append(f"Rounds moyens : {r.avg_rounds:.1f}")
|
||||
lines.append("")
|
||||
lines.append("Attrition moyenne par combatant :")
|
||||
for cid, attr in r.attrition.items():
|
||||
lines.append(
|
||||
f" {cid} : {attr.hits:.2f} touches · {attr.crits:.2f} critiques · "
|
||||
f"{attr.damage_dealt:.2f} dégâts infligés · {attr.damage_taken:.2f} subis"
|
||||
f" · {attr.healing_done:.2f} soins prodigués"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_danger(danger: DangerMetrics) -> str:
|
||||
"""Render DangerMetrics section."""
|
||||
lines = ["=== Métriques de danger ==="]
|
||||
lines.append(f"runs: {danger.runs}")
|
||||
lines.append(f"p_survives: {danger.p_survives:.2f}")
|
||||
lines.append(f"p_tpk: {danger.p_tpk:.2f}")
|
||||
lines.append(f"p_any_death: {danger.p_any_death:.2f}")
|
||||
lines.append(f"expected_hp_lost: {danger.expected_hp_lost:.1f}")
|
||||
lines.append(f"expected_rounds: {danger.expected_rounds:.1f}")
|
||||
lines.append(f"worst_decile_party_hp: {danger.worst_decile_party_hp:.1f}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_nl_stats(stats: NLPolicyStats) -> str:
|
||||
lines = ["=== Observabilité LLM ==="]
|
||||
lines.append(f"calls: {stats.calls}")
|
||||
lines.append(f"repairs_attempted: {stats.repairs_attempted}")
|
||||
lines.append(f"repairs_succeeded: {stats.repairs_succeeded}")
|
||||
lines.append(f"fallbacks: {stats.fallbacks}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_client_stats(stats: ClientStats) -> str:
|
||||
lines = ["=== Coût LLM ==="]
|
||||
lines.append(f"prompt_tokens: {stats.prompt_tokens}")
|
||||
lines.append(f"completion_tokens: {stats.completion_tokens}")
|
||||
lines.append(f"cost_usd: {stats.cost_usd:.3f}")
|
||||
lines.append(f"cache_hits: {stats.cache_hits}")
|
||||
lines.append(f"cache_misses: {stats.cache_misses}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_matchup(matchup: MatchupBenchmark) -> str:
|
||||
lines = ["=== Robustesse (matchup) ==="]
|
||||
lines.append(f"party: {matchup.party}")
|
||||
lines.append(f"adversary: {matchup.adversary}")
|
||||
lines.append(f"runs: {matchup.runs}")
|
||||
lines.append(f"seed: {matchup.seed}")
|
||||
lines.append(f"party_danger.p_survives: {matchup.party_danger.p_survives:.2f}")
|
||||
lines.append(f"party_danger.p_tpk: {matchup.party_danger.p_tpk:.2f}")
|
||||
lines.append(f"adversary_danger.p_survives: {matchup.adversary_danger.p_survives:.2f}")
|
||||
lines.append(f"adversary_danger.p_tpk: {matchup.adversary_danger.p_tpk:.2f}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_report_prompt(ctx: ReportContext, template: str) -> str:
|
||||
"""Build the complete LLM prompt for the report writer."""
|
||||
sections: list[str] = [template, ""]
|
||||
|
||||
sections.append(_render_encounter(ctx))
|
||||
sections.append("")
|
||||
|
||||
if ctx.danger is not None:
|
||||
sections.append(_render_danger(ctx.danger))
|
||||
sections.append("")
|
||||
|
||||
if ctx.matchup is not None:
|
||||
sections.append(_render_matchup(ctx.matchup))
|
||||
sections.append("")
|
||||
|
||||
if ctx.nl_stats is not None:
|
||||
sections.append(_render_nl_stats(ctx.nl_stats))
|
||||
sections.append("")
|
||||
|
||||
if ctx.client_stats is not None:
|
||||
sections.append(_render_client_stats(ctx.client_stats))
|
||||
sections.append("")
|
||||
|
||||
if sections[-1] == "":
|
||||
sections.pop()
|
||||
|
||||
return "\n".join(sections)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReportBudget:
|
||||
"""Budget limits for a single report generation (integrates with CostBudget).
|
||||
|
||||
Mirrors the ceilings from the architecture doc (section 13):
|
||||
small caps for showcase/vitrine mode — max 1 call per report.
|
||||
"""
|
||||
|
||||
max_calls: int = 1
|
||||
max_total_tokens: int | None = None
|
||||
max_cost_usd: float | None = None
|
||||
|
||||
def to_cost_budget(self) -> CostBudget:
|
||||
"""Convert to the client's CostBudget type."""
|
||||
return CostBudget(
|
||||
max_calls=self.max_calls,
|
||||
max_total_tokens=self.max_total_tokens,
|
||||
max_cost_usd=self.max_cost_usd,
|
||||
)
|
||||
|
||||
def for_retries(self, max_retries: int) -> ReportBudget:
|
||||
"""Return a budget allowing for retries (max_calls = max_retries + 1)."""
|
||||
if self.max_calls == 1:
|
||||
return ReportBudget(
|
||||
max_calls=max_retries + 1,
|
||||
max_total_tokens=self.max_total_tokens,
|
||||
max_cost_usd=self.max_cost_usd,
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReportStats:
|
||||
"""Mutable counters for the LLM report writer."""
|
||||
|
||||
calls: int = 0
|
||||
retries: int = 0
|
||||
fallbacks: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LLMReportPolicy:
|
||||
"""Generate a French narrative report via LLM completion.
|
||||
|
||||
Distinct from NLPolicy: produces free-text narrative (not JSON actions),
|
||||
uses simple retry loop (no repair trail), and tracks separate stats.
|
||||
"""
|
||||
|
||||
client: LLMClient
|
||||
builder: PromptBuilder
|
||||
budget: ReportBudget = field(default_factory=ReportBudget)
|
||||
prompt_name: str = "report"
|
||||
max_retries: int = 1
|
||||
stats: ReportStats = field(default_factory=ReportStats)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.max_retries < 0:
|
||||
msg = f"max_retries must be >= 0, got {self.max_retries}"
|
||||
raise ValueError(msg)
|
||||
expected = self.builder.prompt_version(self.prompt_name)
|
||||
if self.client.prompt_version != expected:
|
||||
msg = (
|
||||
f"client prompt_version {self.client.prompt_version!r} does not "
|
||||
f"match pack {expected!r} for {self.prompt_name!r}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
# Apply the report budget (adjusted for retries) to the client
|
||||
if self.client._budget is None: # noqa: SLF001 - private access required for budget injection # pyright: ignore[reportPrivateUsage]
|
||||
adjusted_budget = self.budget.for_retries(self.max_retries)
|
||||
self.client._budget = adjusted_budget.to_cost_budget() # noqa: SLF001 - private access required for budget injection # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
def __call__(self, ctx: ReportContext) -> str:
|
||||
prompt = build_report_prompt(ctx, self.builder.pack.get(self.prompt_name))
|
||||
retries = 0
|
||||
while True:
|
||||
self.stats.calls += 1
|
||||
try:
|
||||
return self.client.complete(prompt)
|
||||
except BudgetExceededError:
|
||||
# Budget exceeded during retry - treat as fallback
|
||||
self.stats.fallbacks += 1
|
||||
return ""
|
||||
except Exception: # noqa: BLE001 - client may raise various exceptions
|
||||
if retries >= self.max_retries:
|
||||
self.stats.fallbacks += 1
|
||||
return ""
|
||||
retries += 1
|
||||
self.stats.retries += 1
|
||||
|
||||
|
||||
def format_llm_report(ctx: ReportContext, llm_text: str) -> str:
|
||||
"""Combine the factual balance report with the LLM narrative section.
|
||||
|
||||
The result is the complete French report printed to stdout and/or written
|
||||
to a file via `--output`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ctx: The populated `ReportContext` with all available data.
|
||||
llm_text: The narrative text returned by `LLMReportPolicy` (may be empty).
|
||||
|
||||
Returns
|
||||
-------
|
||||
Complete French report as a single string.
|
||||
"""
|
||||
factual = _render_encounter(ctx)
|
||||
if not llm_text.strip():
|
||||
return factual
|
||||
|
||||
return f"{factual}\n\n=== Récit d'équilibrage (LLM) ===\n{llm_text.strip()}"
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
Vous êtes le narrateur pour un simulateur de combat Pathfinder 1e.
|
||||
|
||||
Vous recevez un rapport de bataille structuré contenant :
|
||||
- Le rapport d'équilibrage factuel (taux de victoire, bandes de confiance 3σ, attrition)
|
||||
- Les métriques de danger (probabilité de survie, risque TPK, perte de PV attendue, queue pire décile)
|
||||
- La cartographie de robustesse optionnelle (matchups, carte de danger, frontière, sensibilité)
|
||||
- L'observabilité LLM (appels, réparations, fallbacks, usage tokens, coût)
|
||||
|
||||
Rédigez un résumé narratif concis en français pour le maître de jeu. Ton : analytique, évocateur, ancré dans les chiffres. N'inventez PAS de chiffres — citez les statistiques fournies textuellement. PAS de formatage markdown. Texte brut, français uniquement.
|
||||
|
||||
Structurez votre réponse ainsi :
|
||||
1. **Bilan d'équilibrage** — 2-3 phrases synthétisant les taux de victoire et leur signification.
|
||||
2. **Danger et attrition** — 2-3 phrases sur survie/TPK/perte PV/pire décile.
|
||||
3. **Robustesse** (si données présentes) — 1-2 phrases sur matchup/carte de danger/frontière/sensibilité.
|
||||
4. **Observabilité LLM** (si données présentes) — 1 phrase sur appels/réparations/coût.
|
||||
@@ -0,0 +1,62 @@
|
||||
"""CLI integration test for --llm-report flag (P5.3 T5)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.cli import main
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
|
||||
|
||||
def _argv_common() -> list[str]:
|
||||
return [
|
||||
"--map", str(MAP),
|
||||
"--side", "players", str(ROOT / "data" / "monsters" / "goblin.json"),
|
||||
"--side", "monsters", str(ROOT / "data" / "monsters" / "orc.json"),
|
||||
]
|
||||
|
||||
|
||||
def test_cli_llm_report_mutually_exclusive_with_transcript() -> None:
|
||||
"""--llm-report and --transcript are mutually exclusive."""
|
||||
argv = [*_argv_common(), "--llm-report", "--transcript"]
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
main(argv)
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
|
||||
def test_cli_output_requires_llm_report() -> None:
|
||||
"""--output requires --llm-report."""
|
||||
argv = [*_argv_common(), "--output", "report.md"]
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
main(argv)
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
|
||||
def test_cli_llm_report_requires_api_key_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""--llm-report fails gracefully when PF1E_LLM_API_KEY not set."""
|
||||
monkeypatch.delenv("PF1E_LLM_API_KEY", raising=False)
|
||||
monkeypatch.delenv("PF1E_PROMPT_PACK_DIR", raising=False)
|
||||
|
||||
argv = [*_argv_common(), "--llm-report"]
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
main(argv)
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
|
||||
def test_cli_llm_report_requires_prompt_pack_dir(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""--llm-report fails gracefully when PF1E_PROMPT_PACK_DIR not set."""
|
||||
monkeypatch.setenv("PF1E_LLM_API_KEY", "test-key")
|
||||
monkeypatch.delenv("PF1E_PROMPT_PACK_DIR", raising=False)
|
||||
|
||||
argv = [*_argv_common(), "--llm-report"]
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
main(argv)
|
||||
assert exc_info.value.code == 2
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Tests for format_llm_report (P5.3 T4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pf1e_simulator.report import build_report_context, format_llm_report
|
||||
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
|
||||
|
||||
|
||||
def _make_encounter() -> EncounterReport:
|
||||
return EncounterReport(
|
||||
runs=100,
|
||||
seed=1,
|
||||
wins={"players": 65, "monsters": 33},
|
||||
draws=2,
|
||||
avg_rounds=7.2,
|
||||
attrition={
|
||||
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
|
||||
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_format_llm_report_empty_narrative() -> None:
|
||||
"""Empty LLM text returns only the factual report."""
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
result = format_llm_report(ctx, "")
|
||||
|
||||
assert "=== Rapport d'équilibrage ===" in result
|
||||
assert "Victoires players : 65.0%" in result
|
||||
assert "Récit d'équilibrage" not in result
|
||||
|
||||
|
||||
def test_format_llm_report_whitespace_only_narrative() -> None:
|
||||
"""Whitespace-only LLM text returns only the factual report."""
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
result = format_llm_report(ctx, " \n \t ")
|
||||
|
||||
assert "=== Rapport d'équilibrage ===" in result
|
||||
assert "Récit d'équilibrage" not in result
|
||||
|
||||
|
||||
def test_format_llm_report_with_narrative() -> None:
|
||||
"""Non-empty LLM text appends narrative section after factual report."""
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
narrative = "Le gobelin a surpris l'orc par sa vitesse."
|
||||
result = format_llm_report(ctx, narrative)
|
||||
|
||||
assert "=== Rapport d'équilibrage ===" in result
|
||||
assert "Victoires players : 65.0%" in result
|
||||
assert "=== Récit d'équilibrage (LLM) ===" in result
|
||||
assert narrative in result
|
||||
# factual comes before narrative
|
||||
factual_end = result.index("=== Récit d'équilibrage (LLM) ===")
|
||||
assert "Victoires players" in result[:factual_end]
|
||||
|
||||
|
||||
def test_format_llm_report_strips_narrative_whitespace() -> None:
|
||||
"""Leading/trailing whitespace in narrative is stripped."""
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
result = format_llm_report(ctx, " \n Texte narratif. \n ")
|
||||
|
||||
assert result.endswith("Texte narratif.")
|
||||
assert "=== Récit d'équilibrage (LLM) ===\nTexte narratif." in result
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Tests for LLMReportPolicy (P5.3 T3)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.client import FakeLLMClient
|
||||
from pf1e_simulator.prompt import PromptPack
|
||||
from pf1e_simulator.prompt_builder import PromptBuilder
|
||||
from pf1e_simulator.report import LLMReportPolicy, build_report_context
|
||||
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
|
||||
|
||||
def _make_encounter() -> EncounterReport:
|
||||
return EncounterReport(
|
||||
runs=100,
|
||||
seed=1,
|
||||
wins={"players": 65, "monsters": 33},
|
||||
draws=2,
|
||||
avg_rounds=7.2,
|
||||
attrition={
|
||||
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
|
||||
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_policy(
|
||||
script: list[str],
|
||||
*,
|
||||
max_retries: int = 1,
|
||||
) -> LLMReportPolicy:
|
||||
pack = PromptPack.load(FIXTURE_DIR)
|
||||
builder = PromptBuilder(pack)
|
||||
expected_version = pack.prompt_version("report")
|
||||
client = FakeLLMClient(script=script, prompt_version=expected_version) # type: ignore[arg-type]
|
||||
return LLMReportPolicy(
|
||||
client=client, builder=builder, prompt_name="report", max_retries=max_retries
|
||||
)
|
||||
|
||||
|
||||
def test_llm_report_policy_returns_text() -> None:
|
||||
"""Policy returns the scripted text from FakeLLMClient."""
|
||||
policy = _make_policy(script=["Rapport narratif de test."])
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport narratif de test."
|
||||
assert policy.stats.calls == 1
|
||||
assert policy.stats.retries == 0
|
||||
assert policy.stats.fallbacks == 0
|
||||
|
||||
|
||||
def test_llm_report_policy_retry_on_exception() -> None:
|
||||
"""Policy retries on exception and increments retries counter."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport après retry."
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 0
|
||||
|
||||
|
||||
def test_llm_report_policy_fallback_after_max_retries() -> None:
|
||||
"""Policy returns empty string and increments fallbacks after max retries."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom"), Exception("boom2")], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(encounter=_make_encounter(), spec_name="test")
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == ""
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 1
|
||||
|
||||
|
||||
def test_llm_report_policy_prompt_version_mismatch_raises() -> None:
|
||||
"""Policy rejects client with mismatched prompt_version."""
|
||||
pack = PromptPack.load(FIXTURE_DIR)
|
||||
builder = PromptBuilder(pack)
|
||||
expected_version = pack.prompt_version("report")
|
||||
client = FakeLLMClient(script=["x"], prompt_version=expected_version)
|
||||
client.prompt_version = "wrong-version"
|
||||
|
||||
with pytest.raises(ValueError, match="prompt_version"):
|
||||
LLMReportPolicy(client=client, builder=builder, prompt_name="report")
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Tests for ReportBudget and budget exceeded handling (P5.3 T6)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pf1e_simulator.client import CostBudget, FakeLLMClient
|
||||
from pf1e_simulator.prompt import PromptPack
|
||||
from pf1e_simulator.prompt_builder import PromptBuilder
|
||||
from pf1e_simulator.report import (
|
||||
LLMReportPolicy,
|
||||
ReportBudget,
|
||||
build_report_context,
|
||||
)
|
||||
from pf1e_simulator.runner import EncounterReport
|
||||
|
||||
FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
|
||||
|
||||
def _make_policy(
|
||||
script: list[str],
|
||||
*,
|
||||
max_retries: int = 1,
|
||||
budget: ReportBudget | None = None,
|
||||
) -> LLMReportPolicy:
|
||||
pack = PromptPack.load(FIXTURE_DIR)
|
||||
builder = PromptBuilder(pack)
|
||||
client = FakeLLMClient(script=script, prompt_version=pack.prompt_version("report")) # type: ignore[arg-type]
|
||||
return LLMReportPolicy(
|
||||
client=client,
|
||||
builder=builder,
|
||||
prompt_name="report",
|
||||
max_retries=max_retries,
|
||||
budget=budget or ReportBudget(),
|
||||
)
|
||||
|
||||
|
||||
def test_report_budget_default_allows_one_call() -> None:
|
||||
"""Default budget allows exactly 1 successful call."""
|
||||
policy = _make_policy(script=["Rapport de test."])
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport de test."
|
||||
assert policy.stats.calls == 1
|
||||
|
||||
|
||||
def test_report_budget_exceeded_on_second_call() -> None:
|
||||
"""Budget exceeded when attempting a second call."""
|
||||
policy = _make_policy(
|
||||
script=["First call.", "Second call (should not happen)."],
|
||||
max_retries=0, # No retries
|
||||
)
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
# First call succeeds
|
||||
result1 = policy(ctx)
|
||||
assert result1 == "First call."
|
||||
assert policy.stats.calls == 1
|
||||
|
||||
# Second call should exceed budget and return empty string (fallback)
|
||||
result2 = policy(ctx)
|
||||
assert result2 == ""
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.fallbacks == 1
|
||||
|
||||
|
||||
def test_report_budget_for_retries_allows_retry() -> None:
|
||||
"""Budget adjusted for retries allows the retry to succeed."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom"), "Rapport après retry."], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == "Rapport après retry."
|
||||
assert policy.stats.calls == 2
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 0
|
||||
|
||||
|
||||
def test_report_budget_for_retries_exceeded() -> None:
|
||||
"""Budget still exceeded after max retries exhausted."""
|
||||
policy = _make_policy(
|
||||
script=[Exception("boom1"), Exception("boom2"), "Should not reach here."], # type: ignore[list-item]
|
||||
max_retries=1,
|
||||
)
|
||||
ctx = build_report_context(
|
||||
encounter=EncounterReport(
|
||||
runs=10, seed=1, wins={"players": 5, "monsters": 5}, draws=0, avg_rounds=5.0,
|
||||
attrition={}
|
||||
),
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
result = policy(ctx)
|
||||
|
||||
assert result == ""
|
||||
assert policy.stats.calls == 2 # initial + 1 retry (budget allows max_retries + 1 = 2)
|
||||
assert policy.stats.retries == 1
|
||||
assert policy.stats.fallbacks == 1
|
||||
|
||||
|
||||
def test_report_budget_to_cost_budget() -> None:
|
||||
"""ReportBudget converts to CostBudget correctly."""
|
||||
budget = ReportBudget(max_calls=2, max_total_tokens=1000, max_cost_usd=0.01)
|
||||
cost_budget = budget.to_cost_budget()
|
||||
|
||||
assert isinstance(cost_budget, CostBudget)
|
||||
assert cost_budget.max_calls == 2
|
||||
assert cost_budget.max_total_tokens == 1000
|
||||
assert cost_budget.max_cost_usd == 0.01
|
||||
|
||||
|
||||
def test_report_budget_for_retries_adjusts_max_calls() -> None:
|
||||
"""for_retries adjusts max_calls to max_retries + 1."""
|
||||
budget = ReportBudget(max_calls=1)
|
||||
adjusted = budget.for_retries(2)
|
||||
|
||||
assert adjusted.max_calls == 3 # 2 retries + 1 initial
|
||||
assert adjusted.max_total_tokens is None
|
||||
assert adjusted.max_cost_usd is None
|
||||
|
||||
|
||||
def test_report_budget_for_retries_preserves_other_limits() -> None:
|
||||
"""for_retries preserves token and cost limits."""
|
||||
budget = ReportBudget(max_calls=1, max_total_tokens=500, max_cost_usd=0.005)
|
||||
adjusted = budget.for_retries(3)
|
||||
|
||||
assert adjusted.max_calls == 4
|
||||
assert adjusted.max_total_tokens == 500
|
||||
assert adjusted.max_cost_usd == 0.005
|
||||
|
||||
|
||||
def test_report_budget_for_retries_preserves_non_default_max_calls() -> None:
|
||||
"""for_retries preserves non-default max_calls."""
|
||||
budget = ReportBudget(max_calls=5)
|
||||
adjusted = budget.for_retries(2)
|
||||
|
||||
assert adjusted.max_calls == 5 # unchanged because it wasn't the default 1
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Tests for ReportContext and build_report_context (P5.3 T1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from pf1e_simulator.client import ClientStats
|
||||
from pf1e_simulator.loaders import load_combatant
|
||||
from pf1e_simulator.map import load_map
|
||||
from pf1e_simulator.nl_policy import NLPolicyStats
|
||||
from pf1e_simulator.objectives import DangerMetrics
|
||||
from pf1e_simulator.report import ReportContext, build_report_context
|
||||
from pf1e_simulator.runner import EncounterSpec, Side, run_encounter
|
||||
from pf1e_simulator.spells import SPELLS_DIR, load_spell_registry
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAP = ROOT / "data" / "maps" / "sample_arena.yaml"
|
||||
|
||||
|
||||
def _c(path: str):
|
||||
return load_combatant(ROOT / "data" / path)
|
||||
|
||||
|
||||
def _spec() -> EncounterSpec:
|
||||
"""Fast 1v1 (goblin vs orc) on the sample arena."""
|
||||
return EncounterSpec(
|
||||
map=load_map(MAP),
|
||||
round_cap=20,
|
||||
sides=(
|
||||
Side(name="players", combatants=(_c("monsters/goblin.json"),), zone="A"),
|
||||
Side(name="monsters", combatants=(_c("monsters/orc.json"),), zone="B"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_build_report_context_minimal() -> None:
|
||||
"""Minimal context: only EncounterReport, no RunOutcome."""
|
||||
registry = load_spell_registry(SPELLS_DIR)
|
||||
spec = _spec()
|
||||
report = run_encounter(spec, runs=10, seed=1, spell_registry=registry)
|
||||
|
||||
ctx = build_report_context(
|
||||
encounter=report,
|
||||
spec_name="goblin-vs-orc",
|
||||
)
|
||||
|
||||
assert isinstance(ctx, ReportContext)
|
||||
assert ctx.encounter is report
|
||||
assert ctx.spec_name == "goblin-vs-orc"
|
||||
assert ctx.danger is None
|
||||
assert ctx.nl_stats is None
|
||||
assert ctx.client_stats is None
|
||||
assert ctx.matchup is None
|
||||
assert ctx.danger_map is None
|
||||
assert ctx.frontier_value is None
|
||||
assert ctx.sensitivity is None
|
||||
|
||||
|
||||
def test_build_report_context_with_outcomes() -> None:
|
||||
"""Context with RunOutcome -> DangerMetrics computed for the party."""
|
||||
registry = load_spell_registry(SPELLS_DIR)
|
||||
spec = _spec()
|
||||
report, outcomes = run_encounter(
|
||||
spec, runs=100, seed=1, spell_registry=registry, collect_runs=True
|
||||
)
|
||||
|
||||
ctx = build_report_context(
|
||||
encounter=report,
|
||||
spec_name="goblin-vs-orc",
|
||||
outcomes=outcomes,
|
||||
party="players",
|
||||
)
|
||||
|
||||
assert ctx.danger is not None
|
||||
assert isinstance(ctx.danger, DangerMetrics)
|
||||
assert ctx.danger.runs == 100
|
||||
# p_survives, p_tpk, p_any_death are probabilities in [0, 1]
|
||||
assert 0.0 <= ctx.danger.p_survives <= 1.0
|
||||
assert 0.0 <= ctx.danger.p_tpk <= 1.0
|
||||
assert 0.0 <= ctx.danger.p_any_death <= 1.0
|
||||
assert ctx.danger.expected_hp_lost >= 0.0
|
||||
assert ctx.danger.expected_rounds > 0.0
|
||||
assert ctx.danger.worst_decile_party_hp >= 0.0
|
||||
|
||||
|
||||
def test_build_report_context_with_all_optional_fields() -> None:
|
||||
"""Context with all optional fields populated."""
|
||||
registry = load_spell_registry(SPELLS_DIR)
|
||||
spec = _spec()
|
||||
report, outcomes = run_encounter(
|
||||
spec, runs=100, seed=1, spell_registry=registry, collect_runs=True
|
||||
)
|
||||
|
||||
nl_stats = NLPolicyStats(calls=5, repairs_attempted=1, repairs_succeeded=1, fallbacks=0)
|
||||
client_stats = ClientStats(
|
||||
calls=5,
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=200,
|
||||
cost_usd=0.001,
|
||||
cache_hits=0,
|
||||
cache_misses=5,
|
||||
)
|
||||
|
||||
ctx = build_report_context(
|
||||
encounter=report,
|
||||
spec_name="goblin-vs-orc",
|
||||
outcomes=outcomes,
|
||||
party="players",
|
||||
nl_stats=nl_stats,
|
||||
client_stats=client_stats,
|
||||
)
|
||||
|
||||
assert ctx.nl_stats is nl_stats
|
||||
assert ctx.client_stats is client_stats
|
||||
assert ctx.nl_stats.calls == 5 # type: ignore[reportOptionalMemberAccess]
|
||||
assert ctx.nl_stats.repairs_attempted == 1 # type: ignore[reportOptionalMemberAccess]
|
||||
assert ctx.nl_stats.repairs_succeeded == 1 # type: ignore[reportOptionalMemberAccess]
|
||||
assert ctx.nl_stats.fallbacks == 0 # type: ignore[reportOptionalMemberAccess]
|
||||
assert ctx.client_stats.prompt_tokens == 1000 # type: ignore[reportOptionalMemberAccess]
|
||||
assert ctx.client_stats.cost_usd == 0.001 # type: ignore[reportOptionalMemberAccess]
|
||||
|
||||
|
||||
def test_report_context_immutable() -> None:
|
||||
"""ReportContext is a frozen dataclass (immutable)."""
|
||||
registry = load_spell_registry(SPELLS_DIR)
|
||||
spec = _spec()
|
||||
report = run_encounter(spec, runs=5, seed=1, spell_registry=registry)
|
||||
|
||||
ctx = build_report_context(
|
||||
encounter=report,
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
# Frozen dataclass -> assignment raises FrozenInstanceError
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
ctx.encounter = None # type: ignore[misc]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Tests for report prompt pack (P5.3 T2)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pf1e_simulator.prompt import PromptPack
|
||||
from pf1e_simulator.report import build_report_context, build_report_prompt
|
||||
from pf1e_simulator.runner import CombatantAttrition, EncounterReport
|
||||
|
||||
|
||||
def test_report_prompt_pack_version_stable() -> None:
|
||||
"""Prompt pack version hash is stable for the fixture."""
|
||||
pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
pack = PromptPack.load(pack_dir)
|
||||
|
||||
report_version = pack.prompt_version("report")
|
||||
assert report_version.startswith("0.1.0-test:report:")
|
||||
assert len(report_version.split(":")[2]) == 12 # hash12
|
||||
|
||||
|
||||
def test_report_prompt_renders() -> None:
|
||||
"""build_report_prompt renders the context correctly."""
|
||||
pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
pack = PromptPack.load(pack_dir)
|
||||
template = pack.get("report")
|
||||
|
||||
encounter = EncounterReport(
|
||||
runs=100,
|
||||
seed=1,
|
||||
wins={"players": 65, "monsters": 33},
|
||||
draws=2,
|
||||
avg_rounds=7.2,
|
||||
attrition={
|
||||
"goblin": CombatantAttrition(2.45, 0.12, 18.30, 4.20, 0.00),
|
||||
"orc": CombatantAttrition(3.20, 0.25, 28.40, 15.60, 0.00),
|
||||
},
|
||||
)
|
||||
|
||||
ctx = build_report_context(
|
||||
encounter=encounter,
|
||||
spec_name="test",
|
||||
)
|
||||
|
||||
prompt = build_report_prompt(ctx, template)
|
||||
|
||||
# Should contain the system prompt text
|
||||
assert "Vous êtes le narrateur" in prompt
|
||||
assert "Bilan d'équilibrage" in prompt
|
||||
assert "Danger et attrition" in prompt
|
||||
# Should contain the rendered encounter data
|
||||
assert "Victoires players : 65.0%" in prompt or "Victoires players : 65" in prompt
|
||||
assert "goblin" in prompt
|
||||
assert "orc" in prompt
|
||||
# Should contain attrition
|
||||
assert "Attrition moyenne par combatant" in prompt
|
||||
assert "touches" in prompt
|
||||
|
||||
|
||||
def test_report_prompt_version_different_from_nl_default() -> None:
|
||||
"""report prompt_version differs from nl_default."""
|
||||
pack_dir = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
||||
pack = PromptPack.load(pack_dir)
|
||||
|
||||
report_version = pack.prompt_version("report")
|
||||
nl_version = pack.prompt_version("nl_default")
|
||||
assert report_version != nl_version
|
||||
Reference in New Issue
Block a user