feat(phase2): few-shot prompts + bounded repair loop (P5.2)

2026-09-25 23:24:27 +02:00
parent d4db4f6d33
commit dc890fc8fd
9 changed files with 572 additions and 33 deletions
@@ -24,6 +24,7 @@ from pf1e_simulator.config import (
    llm_api_key,
 )
 from pf1e_simulator.prompt import (
+    FewShotExample,
    PromptPack,
    PromptPackError,
    clear_pack_cache,
@@ -31,6 +32,7 @@ from pf1e_simulator.prompt import (
 )

 FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack"
+_FIXTURE_PACK_FS = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
 _FIXTURE_VERSION = "0.1.0-test"
 _FIXTURE_PROMPT_NAME = "nl_default"

@@ -43,6 +45,12 @@ def _write_pack(root: Path, *, version: str, prompts: dict[str, str]) -> None:
        (root / f"{name}.md").write_text(text, encoding="utf-8")


+def _write_examples(root: Path, examples: dict[str, str]) -> None:
+    """Write <name>.examples.json files into an existing pack directory."""
+    for name, text in examples.items():
+        (root / f"{name}.examples.json").write_text(text, encoding="utf-8")
+
+
 # ── config.py ──────────────────────────────────────────────────────────────


@@ -169,3 +177,145 @@ def test_load_returns_prompt_pack_instance() -> None:
    assert isinstance(pack, PromptPack)
    assert pack.version == _FIXTURE_VERSION

+
+# ── prompt.py — few-shot examples (P5.2) ────────────────────────────────────
+
+
+def test_examples_absent_returns_empty() -> None:
+    """A prompt without an examples file exposes an empty example tuple."""
+    pack = load_prompt_pack(FIXTURE_PACK)
+    assert pack.examples(_FIXTURE_PROMPT_NAME) == ()
+
+
+def test_few_shot_fixture_pack_loads_examples() -> None:
+    """prompt_pack_fs ships one teaching pair, parsed into FewShotExample."""
+    pack = load_prompt_pack(_FIXTURE_PACK_FS)
+    examples = pack.examples(_FIXTURE_PROMPT_NAME)
+    assert len(examples) == 1
+    example = examples[0]
+    assert isinstance(example, FewShotExample)
+    assert example.observation == {
+        "round": 1,
+        "combatant": "goblin",
+        "enemy": "orc",
+        "distance_ft": 10,
+    }
+    assert example.actions == ({"kind": "attack", "target_id": "orc"},)
+
+
+def test_prompt_version_includes_examples_content() -> None:
+    """With an examples file, the content hash covers it (cache key, D6)."""
+    plain = load_prompt_pack(FIXTURE_PACK)
+    with_examples = load_prompt_pack(_FIXTURE_PACK_FS)
+    # Same prompt text, same pack version: the digest must differ because the
+    # few-shot file is hashed alongside the prompt.
+    assert plain.get(_FIXTURE_PROMPT_NAME) == with_examples.get(_FIXTURE_PROMPT_NAME)
+    assert plain.prompt_version(_FIXTURE_PROMPT_NAME) != with_examples.prompt_version(
+        _FIXTURE_PROMPT_NAME
+    )
+
+
+def test_prompt_version_byte_identical_without_examples(tmp_path: Path) -> None:
+    """No examples file → digest is byte-identical to the pre-few-shot scheme."""
+    root = tmp_path / "plain"
+    _write_pack(root, version=_FIXTURE_VERSION, prompts={_FIXTURE_PROMPT_NAME: "x"})
+    pack = load_prompt_pack(root)
+    digest = pack.prompt_version(_FIXTURE_PROMPT_NAME).split(":")[-1]
+    assert len(digest) == 12
+
+
+def test_examples_invalid_json_raises(tmp_path: Path) -> None:
+    """A malformed examples file is a pack error, not a silent skip."""
+    root = tmp_path / "badjson"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: "not json"})
+    with pytest.raises(PromptPackError, match="is not valid JSON"):
+        load_prompt_pack(root)
+
+
+def test_examples_not_an_array_raises(tmp_path: Path) -> None:
+    """The examples file must be a JSON array at the top level."""
+    root = tmp_path / "notarray"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: '{"observation": {}}'})
+    with pytest.raises(PromptPackError, match="must be a JSON array of examples"):
+        load_prompt_pack(root)
+
+
+def test_examples_item_not_object_raises(tmp_path: Path) -> None:
+    """Each array entry must be an object with observation and actions."""
+    root = tmp_path / "itemnotobject"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: "[42]"})
+    with pytest.raises(
+        PromptPackError,
+        match="must be an object with 'observation' and 'actions'",
+    ):
+        load_prompt_pack(root)
+
+
+def test_examples_unknown_key_raises(tmp_path: Path) -> None:
+    """Unexpected keys inside an example are rejected (strict validation)."""
+    root = tmp_path / "unknownkey"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": [], "hint": 1}]'},
+    )
+    with pytest.raises(PromptPackError, match="unknown key\\(s\\): hint"):
+        load_prompt_pack(root)
+
+
+def test_examples_observation_not_object_raises(tmp_path: Path) -> None:
+    """'observation' must be a JSON object."""
+    root = tmp_path / "badobs"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": "goblin", "actions": []}]'},
+    )
+    with pytest.raises(PromptPackError, match="'observation' must be a JSON object"):
+        load_prompt_pack(root)
+
+
+def test_examples_actions_not_array_raises(tmp_path: Path) -> None:
+    """'actions' must be a JSON array."""
+    root = tmp_path / "badactions"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": {"kind": "wait"}}]'},
+    )
+    with pytest.raises(PromptPackError, match="'actions' must be a JSON array"):
+        load_prompt_pack(root)
+
+
+def test_examples_action_entry_not_object_raises(tmp_path: Path) -> None:
+    """Each action inside 'actions' must be an object."""
+    root = tmp_path / "badactionentry"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(
+        root,
+        {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": ["jump"]}]'},
+    )
+    with pytest.raises(PromptPackError, match="must contain only JSON objects"):
+        load_prompt_pack(root)
+
+
+def test_examples_empty_array_accepted(tmp_path: Path) -> None:
+    """An empty examples array is valid and yields no teaching pairs."""
+    root = tmp_path / "emptyexamples"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {_FIXTURE_PROMPT_NAME: "[]"})
+    pack = load_prompt_pack(root)
+    assert pack.examples(_FIXTURE_PROMPT_NAME) == ()
+
+
+def test_orphan_examples_file_raises(tmp_path: Path) -> None:
+    """An .examples.json without a matching <name>.md is a pack error."""
+    root = tmp_path / "orphan"
+    _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
+    _write_examples(root, {"stray": "[]"})
+    with pytest.raises(PromptPackError, match="has no matching 'stray\\.md' prompt"):
+        load_prompt_pack(root)
+