feat(phase2): few-shot prompts + bounded repair loop (P5.2)

tests/fixtures/prompt_pack_fs/nl_default.examples.json

@@ -0,0 +1,6 @@
+[
+  {
+    "observation": {"round": 1, "combatant": "goblin", "enemy": "orc", "distance_ft": 10},
+    "actions": [{"kind": "attack", "target_id": "orc"}]
+  }
+]

tests/fixtures/prompt_pack_fs/nl_default.md

@@ -0,0 +1,5 @@
+You are the tactical director for a Pathfinder 1e battle.
+
+You see the current battle state as a compact observation view, followed by
+the list of plausible actions the acting combatant could take.
+Respond with the JSON action sequence described in the schema.

tests/fixtures/prompt_pack_fs/version.txt

@@ -0,0 +1 @@
+0.1.0-test

tests/test_phase2_nl_policy.py

@@ -3,17 +3,28 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import pytest
 
-from pf1e_simulator.client import FakeLLMClient, ResponseCache, ScriptExhaustedError
+from pf1e_simulator.client import (
+    ClientParams,
+    Completion,
+    FakeLLMClient,
+    ResponseCache,
+    ScriptExhaustedError,
+)
 from pf1e_simulator.combat import Action, Observation
 from pf1e_simulator.nl_policy import (
     NLPolicy,
     NLPolicyError,
+    NLPolicyStats,
     fallback_default,
     parse_actions,
 )
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
 from pf1e_simulator.prompt import PromptPack
 from pf1e_simulator.prompt_builder import PromptBuilder
 
@@ -51,8 +62,26 @@ def _make_client(script: list[str], *, builder: PromptBuilder, **kwargs) -> Fake
     )
 
 
-def _make_policy(client: FakeLLMClient, builder: PromptBuilder) -> NLPolicy:
-    return NLPolicy(client=client, builder=builder)
+def _make_policy(
+    client: FakeLLMClient, builder: PromptBuilder, *, max_repairs: int = 1
+) -> NLPolicy:
+    return NLPolicy(client=client, builder=builder, max_repairs=max_repairs)
+
+
+class _RecordingClient(FakeLLMClient):
+    """Scripted client that records every prompt sent to the model.
+
+    The repair-loop tests assert on the accumulated error trail, so the
+    prompts themselves must be observable.
+    """
+
+    def __init__(self, script: Sequence[str], *, prompt_version: str) -> None:
+        super().__init__(script=script, prompt_version=prompt_version)
+        self.prompts: list[str] = []
+
+    def _call_once(self, prompt: str, *, params: ClientParams) -> Completion:
+        self.prompts.append(prompt)
+        return super()._call_once(prompt, params=params)
 
 
 @pytest.fixture
@@ -278,3 +307,108 @@ def test_nl_policy_shared_cache_hit() -> None:
     # Second call was a cache hit (no ScriptExhaustedError)
     assert client2.stats.cache_hits == 1
     assert client1.stats.cache_misses == 1
+
+
+# ---- P5.2 repair loop --------------------------------------------------------
+
+def test_nl_policy_stats_defaults_to_zero() -> None:
+    """A fresh NLPolicy reports zero calls until the first decision."""
+    builder = _builder()
+    client = _make_client(['[{"kind": "wait"}]'], builder=builder)
+    policy = _make_policy(client, builder)
+    assert policy.stats == NLPolicyStats()
+    assert policy.stats.calls == 0
+
+
+def test_nl_policy_max_repairs_zero_falls_back_immediately() -> None:
+    """max_repairs=0 disables repair: first bad reply triggers fallback."""
+    builder = _builder()
+    client = _make_client(["bad one"], builder=builder)
+    policy = _make_policy(client, builder, max_repairs=0)
+
+    actions = policy(_obs())
+    assert actions == (Action(kind="total_defense"),)
+    assert policy.stats.calls == 1
+    assert policy.stats.repairs_attempted == 0
+    assert policy.stats.repairs_succeeded == 0
+    assert policy.stats.fallbacks == 1
+
+
+def test_nl_policy_max_repairs_two_repairs_then_succeeds() -> None:
+    """Two invalid replies are repaired, then the valid reply is parsed."""
+    builder = _builder()
+    script = ["bad one", "bad two", '[{"kind": "move", "destination": [2, 3]}]']
+    client = _make_client(script, builder=builder)
+    policy = _make_policy(client, builder, max_repairs=2)
+
+    actions = policy(_obs())
+    assert actions == (Action(kind="move", destination=(2, 3)),)
+    assert policy.stats.calls == 3
+    assert policy.stats.repairs_attempted == 2
+    assert policy.stats.repairs_succeeded == 1
+    assert policy.stats.fallbacks == 0
+
+
+def test_nl_policy_max_repairs_two_then_falls_back() -> None:
+    """Past max_repairs the policy stops repairing and falls back."""
+    builder = _builder()
+    script = ["bad one", "bad two", "bad three"]
+    client = _make_client(script, builder=builder)
+    policy = _make_policy(client, builder, max_repairs=2)
+
+    actions = policy(_obs())
+    assert actions == (Action(kind="total_defense"),)
+    assert policy.stats.calls == 3
+    assert policy.stats.repairs_attempted == 2
+    assert policy.stats.repairs_succeeded == 0
+    assert policy.stats.fallbacks == 1
+
+
+def test_nl_policy_repair_trail_accumulates() -> None:
+    """Each retry prompt carries the full accumulated error trail."""
+    builder = _builder()
+    script = ["bad one", "bad two", '[{"kind": "wait"}]']
+    client = _RecordingClient(
+        script=script,
+        prompt_version=builder.prompt_version("nl_default"),
+    )
+    policy = _make_policy(client, builder, max_repairs=2)
+
+    actions = policy(_obs())
+    assert actions == (Action(kind="wait"),)
+
+    # First call: bare prompt with no trail.
+    assert len(client.prompts) == 3
+    assert "-- previous reply invalid" not in client.prompts[0]
+    # First retry: trail carries attempt 1 only.
+    assert "attempt 1" in client.prompts[1]
+    assert "attempt 2" not in client.prompts[1]
+    assert "Return a valid JSON action sequence." in client.prompts[1]
+    # Second retry: trail carries attempts 1 AND 2, in order.
+    attempt1 = client.prompts[2].index("-- previous reply invalid (attempt 1) --")
+    attempt2 = client.prompts[2].index("-- previous reply invalid (attempt 2) --")
+    assert attempt1 < attempt2
+    # The base prompt is preserved at the head of every retry.
+    assert client.prompts[1].startswith(client.prompts[0])
+
+
+def test_nl_policy_max_repairs_negative_rejected() -> None:
+    """A negative max_repairs is a configuration error, not a fallback."""
+    builder = _builder()
+    client = _make_client(['[{"kind": "wait"}]'], builder=builder)
+    with pytest.raises(ValueError, match=r"max_repairs must be >= 0, got -1"):
+        _make_policy(client, builder, max_repairs=-1)
+
+
+def test_nl_policy_stats_after_direct_success() -> None:
+    """A clean first-try success leaves the repair counters untouched."""
+    builder = _builder()
+    client = _make_client(['[{"kind": "wait"}]'], builder=builder)
+    policy = _make_policy(client, builder)
+
+    actions = policy(_obs())
+    assert actions == (Action(kind="wait"),)
+    assert policy.stats.calls == 1
+    assert policy.stats.repairs_attempted == 0
+    assert policy.stats.repairs_succeeded == 0
+    assert policy.stats.fallbacks == 0

tests/test_phase2_prompt.py

@@ -24,6 +24,7 @@ from pf1e_simulator.config import (
     llm_api_key,
 )
 from pf1e_simulator.prompt import (
+    FewShotExample,
     PromptPack,
     PromptPackError,
     clear_pack_cache,
@@ -31,6 +32,7 @@ from pf1e_simulator.prompt import (
 )
 
 FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack"
+_FIXTURE_PACK_FS = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
 _FIXTURE_VERSION = "0.1.0-test"
 _FIXTURE_PROMPT_NAME = "nl_default"
 
@@ -43,6 +45,12 @@ def _write_pack(root: Path, *, version: str, prompts: dict[str, str]) -> None:
         (root / f"{name}.md").write_text(text, encoding="utf-8")
 
 
+def _write_examples(root: Path, examples: dict[str, str]) -> None:
+    """Write <name>.examples.json files into an existing pack directory."""
+    for name, text in examples.items():
+        (root / f"{name}.examples.json").write_text(text, encoding="utf-8")
+
+
 # ── config.py ──────────────────────────────────────────────────────────────
 
 
@@ -169,3 +177,145 @@ def test_load_returns_prompt_pack_instance() -> None:
     assert isinstance(pack, PromptPack)
     assert pack.version == _FIXTURE_VERSION
 
+
+# ── prompt.py — few-shot examples (P5.2) ────────────────────────────────────
+
+
+def test_examples_absent_returns_empty() -> None:
+    """A prompt without an examples file exposes an empty example tuple."""
+    pack = load_prompt_pack(FIXTURE_PACK)
+    assert pack.examples(_FIXTURE_PROMPT_NAME) == ()
+
+
+def test_few_shot_fixture_pack_loads_examples() -> None:
+    """prompt_pack_fs ships one teaching pair, parsed into FewShotExample."""
+    pack = load_prompt_pack(_FIXTURE_PACK_FS)
+    examples = pack.examples(_FIXTURE_PROMPT_NAME)
+    assert len(examples) == 1
+    example = examples[0]
+    assert isinstance(example, FewShotExample)
+    assert example.observation == {
+        "round": 1,
+        "combatant": "goblin",
+        "enemy": "orc",
+        "distance_ft": 10,
+    }
+    assert example.actions == ({"kind": "attack", "target_id": "orc"},)
+
+
+def test_prompt_version_includes_examples_content() -> None:
+    """With an examples file, the content hash covers it (cache key, D6)."""
+    plain = load_prompt_pack(FIXTURE_PACK)
+    with_examples = load_prompt_pack(_FIXTURE_PACK_FS)
+    # Same prompt text, same pack version: the digest must differ because the
+    # few-shot file is hashed alongside the prompt.
+    assert plain.get(_FIXTURE_PROMPT_NAME) == with_examples.get(_FIXTURE_PROMPT_NAME)
+    assert plain.prompt_version(_FIXTURE_PROMPT_NAME) != with_examples.prompt_version(
+        _FIXTURE_PROMPT_NAME
+    )
+
+
+def test_prompt_version_byte_identical_without_examples(tmp_path: Path) -> None:
+    """No examples file → digest is byte-identical to the pre-few-shot scheme."""
+    root = tmp_path / "plain"
+    _write_pack(root, version=_FIXTURE_VERSION, prompts={_FIXTURE_PROMPT_NAME: "x"})
+    pack = load_prompt_pack(root)
+    digest = pack.prompt_version(_FIXTURE_PROMPT_NAME).split(":")[-1]
+    assert len(digest) == 12
+
+
+def test_examples_invalid_json_raises(tmp_path: Path) -> None:
+    """A malformed examples file is a pack error, not a silent skip."""
+    root = tmp_path / "badjson"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: "not json"})
+    with pytest.raises(PromptPackError, match="is not valid JSON"):
+        load_prompt_pack(root)
+
+
+def test_examples_not_an_array_raises(tmp_path: Path) -> None:
+    """The examples file must be a JSON array at the top level."""
+    root = tmp_path / "notarray"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: '{"observation": {}}'})
+    with pytest.raises(PromptPackError, match="must be a JSON array of examples"):
+        load_prompt_pack(root)
+
+
+def test_examples_item_not_object_raises(tmp_path: Path) -> None:
+    """Each array entry must be an object with observation and actions."""
+    root = tmp_path / "itemnotobject"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: "[42]"})
+    with pytest.raises(
+        PromptPackError,
+        match="must be an object with 'observation' and 'actions'",
+    ):
+        load_prompt_pack(root)
+
+
+def test_examples_unknown_key_raises(tmp_path: Path) -> None:
+    """Unexpected keys inside an example are rejected (strict validation)."""
+    root = tmp_path / "unknownkey"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": [], "hint": 1}]'},
+    )
+    with pytest.raises(PromptPackError, match="unknown key\\(s\\): hint"):
+        load_prompt_pack(root)
+
+
+def test_examples_observation_not_object_raises(tmp_path: Path) -> None:
+    """'observation' must be a JSON object."""
+    root = tmp_path / "badobs"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": "goblin", "actions": []}]'},
+    )
+    with pytest.raises(PromptPackError, match="'observation' must be a JSON object"):
+        load_prompt_pack(root)
+
+
+def test_examples_actions_not_array_raises(tmp_path: Path) -> None:
+    """'actions' must be a JSON array."""
+    root = tmp_path / "badactions"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": {"kind": "wait"}}]'},
+    )
+    with pytest.raises(PromptPackError, match="'actions' must be a JSON array"):
+        load_prompt_pack(root)
+
+
+def test_examples_action_entry_not_object_raises(tmp_path: Path) -> None:
+    """Each action inside 'actions' must be an object."""
+    root = tmp_path / "badactionentry"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": ["jump"]}]'},
+    )
+    with pytest.raises(PromptPackError, match="must contain only JSON objects"):
+        load_prompt_pack(root)
+
+
+def test_examples_empty_array_accepted(tmp_path: Path) -> None:
+    """An empty examples array is valid and yields no teaching pairs."""
+    root = tmp_path / "emptyexamples"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: "[]"})
+    pack = load_prompt_pack(root)
+    assert pack.examples(_FIXTURE_PROMPT_NAME) == ()
+
+
+def test_orphan_examples_file_raises(tmp_path: Path) -> None:
+    """An .examples.json without a matching <name>.md is a pack error."""
+    root = tmp_path / "orphan"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {"stray": "[]"})
+    with pytest.raises(PromptPackError, match="has no matching 'stray\\.md' prompt"):
+        load_prompt_pack(root)
+

tests/test_phase2_prompt_builder.py

@@ -13,12 +13,14 @@ from pf1e_simulator.nl_policy import get_args as _get_args
 from pf1e_simulator.prompt import PromptPack
 from pf1e_simulator.prompt_builder import (
     PromptBuilder,
+    _render_examples,
     _supported_kinds,
     action_schema,
     render_candidates,
 )
 
 _FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack"
+_FIXTURE_PACK_FS = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
 _FIXTURE_VERSION = "0.1.0-test"
 _FIXTURE_PROMPT_NAME = "nl_default"
 
@@ -221,3 +223,75 @@ def test_action_schema_cross_pin_with_nl_policy_kindvalue() -> None:
     """The kind enum in the schema must match nl_policy.KindValue exactly."""
     schema = action_schema()
     assert schema["properties"]["kind"]["enum"] == list(_get_args(KindValue))
+
+
+# ---- few-shot examples (P5.2) ----------------------------------------------
+
+_FS_PACK = PromptPack.load(_FIXTURE_PACK_FS)
+
+
+def _fs_builder() -> PromptBuilder:
+    return PromptBuilder(pack=_FS_PACK)
+
+
+def test_render_examples_block_format() -> None:
+    """Few-shot pairs render as numbered observation-to-response blocks."""
+    rendered = _render_examples((_FS_PACK.examples(_FIXTURE_PROMPT_NAME)[0],))
+    assert rendered == (
+        "## few-shot examples\n"
+        "### example 1\n"
+        "observation:\n"
+        '{"combatant": "goblin", "distance_ft": 10, "enemy": "orc", "round": 1}\n'
+        "\n"
+        "response:\n"
+        '[{"kind": "attack", "target_id": "orc"}]'
+    )
+
+
+def test_build_without_examples_unchanged() -> None:
+    """Packs without examples keep the pre-few-shot bit-for-bit section order."""
+    plain = _builder().build(_obs())
+    assert "## few-shot examples" not in plain
+    # Template, then observation, then candidates, then schema — in order.
+    sections = [
+        "## observation\n",
+        "## plausible actions\n",
+        "## action schema\n",
+    ]
+    positions = [plain.index(section) for section in sections]
+    assert positions == sorted(positions)
+    # The template comes first, unmodified.
+    assert plain.startswith("You are the tactical director")
+
+
+def test_build_with_examples_inserts_between_template_and_observation() -> None:
+    """The few-shot section sits between the template and the observation."""
+    prompt = _fs_builder().build(_obs())
+    assert "## few-shot examples\n" in prompt
+    assert prompt.index("## few-shot examples\n") < prompt.index("## observation\n")
+    # Template still leads the prompt.
+    assert prompt.startswith("You are the tactical director")
+    # Observation/candidates/schema keep their relative order.
+    assert prompt.index("## observation\n") < prompt.index("## plausible actions\n")
+    assert prompt.index("## plausible actions\n") < prompt.index("## action schema\n")
+
+
+def test_build_with_examples_includes_example_content() -> None:
+    """The rendered example content is embedded verbatim (sorted keys)."""
+    prompt = _fs_builder().build(_obs())
+    assert '{"combatant": "goblin", "distance_ft": 10, "enemy": "orc", "round": 1}' in prompt
+    assert '[{"kind": "attack", "target_id": "orc"}]' in prompt
+    assert "### example 1" in prompt
+
+
+def test_empty_examples_array_no_section(tmp_path: Path) -> None:
+    """An empty examples array yields no few-shot section (not an error)."""
+    root = tmp_path / "emptyfs"
+    root.mkdir(parents=True, exist_ok=True)
+    (root / "version.txt").write_text("1.0.0", encoding="utf-8")
+    (root / "nl_default.md").write_text("T", encoding="utf-8")
+    (root / "nl_default.examples.json").write_text("[]", encoding="utf-8")
+    pack = PromptPack.load(root)
+    assert pack.examples("nl_default") == ()
+    prompt = PromptBuilder(pack=pack).build(_obs())
+    assert "## few-shot examples" not in prompt