diff --git a/src/pf1e_simulator/nl_policy.py b/src/pf1e_simulator/nl_policy.py index 507a40b..f684111 100644 --- a/src/pf1e_simulator/nl_policy.py +++ b/src/pf1e_simulator/nl_policy.py @@ -1,10 +1,16 @@ -"""Natural-language policy: parses LLM JSON replies into Action sequences (P1.3).""" +"""Natural-language policy (P1.3): parses LLM JSON replies into Action sequences. + +P5.2 adds a bounded repair loop: on a parse failure the policy re-asks the +model with an accumulated error trail up to ``max_repairs`` times, then +falls back to the scripted policy; ``NLPolicyStats`` exposes the call and +repair counters for the sweep and the phase-3 report. +""" from __future__ import annotations import json from collections.abc import Callable -from dataclasses import dataclass, fields +from dataclasses import dataclass, field, fields from typing import TYPE_CHECKING, Literal, TypeGuard, get_args from pf1e_simulator.candidates import candidates @@ -233,6 +239,27 @@ def fallback_default(menu: tuple[Action, ...]) -> tuple[Action, ...]: return (Action(kind="wait"),) +# ---- Repair protocol --------------------------------------------------------- + +_REPAIR_INSTRUCTION = "Return a valid JSON action sequence." +_REPAIR_HEADER_TEMPLATE = "-- previous reply invalid (attempt {}) --\n{}" + + +@dataclass +class NLPolicyStats: + """Mutable counters for the LLM decision loop (repair observability). + + The phase-3 report and the sweep pipeline read these counters to + quantify how often the LLM layer repairs invalid replies or falls back + to the scripted policy. + """ + + calls: int = 0 + repairs_attempted: int = 0 + repairs_succeeded: int = 0 + fallbacks: int = 0 + + # ---- NLPolicy ---------------------------------------------------------------- @dataclass(frozen=True) @@ -243,8 +270,13 @@ class NLPolicy: builder: PromptBuilder prompt_name: str = "nl_default" fallback: Fallback = fallback_default + max_repairs: int = 1 + stats: NLPolicyStats = field(default_factory=NLPolicyStats) def __post_init__(self) -> None: + if self.max_repairs < 0: + msg = f"max_repairs must be >= 0, got {self.max_repairs}" + raise ValueError(msg) expected = self.builder.prompt_version(self.prompt_name) if self.client.prompt_version != expected: msg = ( @@ -257,15 +289,25 @@ class NLPolicy: menu = candidates(observation) prompt = self.builder.build(observation, prompt_name=self.prompt_name) reply = self.client.complete(prompt) - try: - return parse_actions(reply) - except NLPolicyError as exc: - retry_prompt = ( - f"{prompt}\n\n-- previous reply invalid --\n{exc}\n" - "Return a valid JSON action sequence." - ) - retry_reply = self.client.complete(retry_prompt) + self.stats.calls += 1 + errors: list[NLPolicyError] = [] + while True: try: - return parse_actions(retry_reply) - except NLPolicyError: - return self.fallback(menu) + actions = parse_actions(reply) + break + except NLPolicyError as exc: + if len(errors) >= self.max_repairs: + self.stats.fallbacks += 1 + return self.fallback(menu) + errors.append(exc) + self.stats.repairs_attempted += 1 + trail = "\n\n".join( + _REPAIR_HEADER_TEMPLATE.format(index, error) + for index, error in enumerate(errors, start=1) + ) + retry_prompt = f"{prompt}\n\n{trail}\n{_REPAIR_INSTRUCTION}" + reply = self.client.complete(retry_prompt) + self.stats.calls += 1 + if errors: + self.stats.repairs_succeeded += 1 + return actions diff --git a/src/pf1e_simulator/prompt.py b/src/pf1e_simulator/prompt.py index 1132e17..0fb13ad 100644 --- a/src/pf1e_simulator/prompt.py +++ b/src/pf1e_simulator/prompt.py @@ -12,22 +12,32 @@ and the sweep reports, so a campaign stays reproducible after a prompt edit Pack layout ----------- / - version.txt # single line: semver or commit hash (mandatory) - .md # one file per prompt name, e.g. ``nl_default.md`` + version.txt # single line: semver or commit hash (mandatory) + .md # one file per prompt name, e.g. ``nl_default.md`` + .examples.json # optional few-shot examples for that prompt: a + # JSON array of ``{"observation": {...}, + # "actions": [{...}, ...]}`` teaching pairs Loading is eager and strict. A missing pack, a missing ``version.txt`` or a missing ``.md`` raises a typed ``PromptPackError`` -- there is no -silent fallback (validation rule, section 14). Unit tests use the minimal -local fixture under ``tests/fixtures/prompt_pack/`` so CI never depends on -the external prompt repository (pinning rule, section 14). +silent fallback (validation rule, section 14). An optional +``.examples.json`` file carries few-shot examples (P5.2): it comes +from the external pack, never from this repo (section 14), and is validated +at load time -- an unreadable, malformed or orphaned examples file is a +pack error, not a silent skip. ``prompt_version`` covers the examples +content alongside the prompt text, so the response-cache key (D6) stays in +sync with any pack edit. Unit tests use the minimal local fixtures under +``tests/fixtures/prompt_pack/`` so CI never depends on the external prompt +repository (pinning rule, section 14). """ from __future__ import annotations import hashlib +import json from dataclasses import dataclass, field from functools import cache -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NoReturn, TypeGuard from pf1e_simulator.config import ENV_PROMPT_PACK_DIR, env_path @@ -35,31 +45,95 @@ if TYPE_CHECKING: from pathlib import Path _PROMPT_SUFFIX = ".md" +_EXAMPLES_SUFFIX = ".examples.json" _VERSION_FILE = "version.txt" _HASH_PREFIX_LEN = 12 +_HASH_SEPARATOR = "\0" +_EXAMPLE_KEYS = frozenset({"observation", "actions"}) class PromptPackError(RuntimeError): """A prompt pack is missing, invalid, or lacks a required prompt.""" -def _pack_error(kind: str, message: str) -> None: +def _pack_error(kind: str, message: str) -> NoReturn: msg = f"{kind}: {message}" raise PromptPackError(msg) +def _is_object_dict(value: object) -> TypeGuard[dict[str, object]]: + return isinstance(value, dict) + + +def _is_object_list(value: object) -> TypeGuard[list[object]]: + return isinstance(value, list) + + +@dataclass(frozen=True) +class FewShotExample: + """One structured few-shot pair: an example observation and a valid reply. + + ``observation`` is a plain JSON object (the pack author composes it, the + builder renders it verbatim); ``actions`` is a JSON array of Action-shaped + objects teaching the exact reply format the parser accepts. + """ + + observation: dict[str, object] + actions: tuple[dict[str, object], ...] + + +def _parse_examples(path: Path, text: str) -> tuple[FewShotExample, ...]: + """Parse and strictly validate one few-shot examples file.""" + try: + raw = json.loads(text) + except json.JSONDecodeError as exc: + msg = f"{path.name} is not valid JSON: {exc}" + _pack_error("invalid pack", msg) + if not _is_object_list(raw): + _pack_error("invalid pack", f"{path.name} must be a JSON array of examples") + examples: list[FewShotExample] = [] + for index, item in enumerate(raw): + if not _is_object_dict(item): + msg = f"{path.name}[{index}] must be an object with 'observation' and 'actions'" + _pack_error("invalid pack", msg) + unknown = sorted(set(item) - _EXAMPLE_KEYS) + if unknown: + msg = f"{path.name}[{index}] has unknown key(s): {', '.join(unknown)}" + _pack_error("invalid pack", msg) + observation = item.get("observation") + actions = item.get("actions") + if not _is_object_dict(observation): + msg = f"{path.name}[{index}] 'observation' must be a JSON object" + _pack_error("invalid pack", msg) + if not _is_object_list(actions): + msg = f"{path.name}[{index}] 'actions' must be a JSON array of objects" + _pack_error("invalid pack", msg) + action_dicts: list[dict[str, object]] = [] + for action in actions: + if not _is_object_dict(action): + msg = f"{path.name}[{index}] 'actions' must contain only JSON objects" + _pack_error("invalid pack", msg) + action_dicts.append(action) + examples.append(FewShotExample(observation=observation, actions=tuple(action_dicts))) + return tuple(examples) + + @dataclass(frozen=True) class PromptPack: """A versioned set of prompts loaded from one external directory. Instances are immutable (frozen), so a loaded pack can be shared by parallel sweep workers without drift. ``_prompts`` maps prompt names - (file stems) to their full text, read at load time. + (file stems) to their full text, read at load time; ``_examples`` maps + prompt names to their parsed few-shot pairs (empty when the pack ships + none) and ``_examples_text`` keeps the raw file bytes for hashing. """ root: Path version: str _prompts: dict[str, str] = field(repr=False) + _examples: dict[str, tuple[FewShotExample, ...]] = field(repr=False) + _examples_text: dict[str, str] = field(repr=False) def get(self, name: str) -> str: """Return the prompt ``name``'s text, or raise ``PromptPackError``.""" @@ -72,10 +146,24 @@ class PromptPack: ) raise PromptPackError(msg) from exc + def examples(self, name: str) -> tuple[FewShotExample, ...]: + """Return the few-shot pairs for ``name`` (empty when none).""" + return self._examples.get(name, ()) + def prompt_version(self, name: str) -> str: - """Pack version plus per-prompt content hash (cache-key component, D6).""" + """Pack version plus per-prompt content hash (cache-key component, D6). + + The digest covers the prompt text and, when the pack ships a + ``.examples.json`` file, that file's bytes too -- any pack edit + invalidates the cache key. Without an examples file the digest is + byte-identical to the pre-few-shot scheme. + """ content = self.get(name) - digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:_HASH_PREFIX_LEN] + examples_text = self._examples_text.get(name) + digest_input = content + if examples_text is not None: + digest_input = f"{content}{_HASH_SEPARATOR}{examples_text}" + digest = hashlib.sha256(digest_input.encode("utf-8")).hexdigest()[:_HASH_PREFIX_LEN] return f"{self.version}:{name}:{digest}" @classmethod @@ -95,7 +183,23 @@ class PromptPack: prompts: dict[str, str] = {} for path in sorted(root.glob(f"*{_PROMPT_SUFFIX}")): prompts[path.stem] = path.read_text(encoding="utf-8") - return cls(root=root, version=version, _prompts=prompts) + examples: dict[str, tuple[FewShotExample, ...]] = {} + examples_text: dict[str, str] = {} + for path in sorted(root.glob(f"*{_EXAMPLES_SUFFIX}")): + stem = path.name.removesuffix(_EXAMPLES_SUFFIX) + if stem not in prompts: + msg = f"{path.name} has no matching '{stem}{_PROMPT_SUFFIX}' prompt" + _pack_error("invalid pack", msg) + text = path.read_text(encoding="utf-8") + examples[stem] = _parse_examples(path, text) + examples_text[stem] = text + return cls( + root=root, + version=version, + _prompts=prompts, + _examples=examples, + _examples_text=examples_text, + ) @cache diff --git a/src/pf1e_simulator/prompt_builder.py b/src/pf1e_simulator/prompt_builder.py index 11ee316..a9084ae 100644 --- a/src/pf1e_simulator/prompt_builder.py +++ b/src/pf1e_simulator/prompt_builder.py @@ -1,9 +1,14 @@ """Prompt builder for the phase-2 LLM layer (P1.4). Builds a deterministic prompt from an ``Observation`` plus the external -prompt pack: template + compact observation + plausible actions + JSON -action schema. The schema is derived from the ``Action`` dataclass via -``typing.get_type_hints`` so it stays in sync with the engine. +prompt pack: template + optional few-shot examples + compact observation + +plausible actions + JSON action schema. The examples (P5.2) come from the +pack's ``.examples.json`` file when present and are mounted between +the template and the live observation, so the query stays the last section +before the reply (few-shot recency) and packs without examples keep the +original section order bit-for-bit. The schema is derived from the +``Action`` dataclass via ``typing.get_type_hints`` so it stays in sync with +the engine. """ from __future__ import annotations @@ -18,7 +23,7 @@ from pf1e_simulator.combat import UNSUPPORTED_ACTION_KINDS, Action, Observation from pf1e_simulator.llm_view import compact_observation if TYPE_CHECKING: - from pf1e_simulator.prompt import PromptPack + from pf1e_simulator.prompt import FewShotExample, PromptPack def _supported_kinds() -> tuple[str, ...]: @@ -152,6 +157,20 @@ def render_candidates(actions: tuple[Action, ...]) -> tuple[str, ...]: return tuple(lines) +def _render_examples(examples: tuple[FewShotExample, ...]) -> str: + """Render few-shot pairs as observation-to-response teaching blocks.""" + blocks: list[str] = [] + for index, example in enumerate(examples): + observation_json = json.dumps(example.observation, sort_keys=True) + actions_json = json.dumps(example.actions, sort_keys=True) + blocks.append( + f"### example {index + 1}\n" + f"observation:\n{observation_json}\n\n" + f"response:\n{actions_json}" + ) + return "## few-shot examples\n" + "\n\n".join(blocks) + + @dataclass(frozen=True) class PromptBuilder: """Builds the complete LLM prompt from a prompt pack and an observation.""" @@ -173,9 +192,13 @@ class PromptBuilder: menu = candidates(observation) actions_text = "\n".join(render_candidates(menu)) schema_text = json.dumps(action_schema(), sort_keys=True, indent=2) - return "\n\n".join([ - template, + sections: list[str] = [template] + examples = self.pack.examples(prompt_name) + if examples: + sections.append(_render_examples(examples)) + sections.extend([ "## observation\n" + obs_text, "## plausible actions\n" + actions_text, "## action schema\n" + schema_text, ]) + return "\n\n".join(sections) diff --git a/tests/fixtures/prompt_pack_fs/nl_default.examples.json b/tests/fixtures/prompt_pack_fs/nl_default.examples.json new file mode 100644 index 0000000..efdaa6d --- /dev/null +++ b/tests/fixtures/prompt_pack_fs/nl_default.examples.json @@ -0,0 +1,6 @@ +[ + { + "observation": {"round": 1, "combatant": "goblin", "enemy": "orc", "distance_ft": 10}, + "actions": [{"kind": "attack", "target_id": "orc"}] + } +] \ No newline at end of file diff --git a/tests/fixtures/prompt_pack_fs/nl_default.md b/tests/fixtures/prompt_pack_fs/nl_default.md new file mode 100644 index 0000000..7dc1287 --- /dev/null +++ b/tests/fixtures/prompt_pack_fs/nl_default.md @@ -0,0 +1,5 @@ +You are the tactical director for a Pathfinder 1e battle. + +You see the current battle state as a compact observation view, followed by +the list of plausible actions the acting combatant could take. +Respond with the JSON action sequence described in the schema. \ No newline at end of file diff --git a/tests/fixtures/prompt_pack_fs/version.txt b/tests/fixtures/prompt_pack_fs/version.txt new file mode 100644 index 0000000..804a184 --- /dev/null +++ b/tests/fixtures/prompt_pack_fs/version.txt @@ -0,0 +1 @@ +0.1.0-test \ No newline at end of file diff --git a/tests/test_phase2_nl_policy.py b/tests/test_phase2_nl_policy.py index ac77c18..479b718 100644 --- a/tests/test_phase2_nl_policy.py +++ b/tests/test_phase2_nl_policy.py @@ -3,17 +3,28 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING import pytest -from pf1e_simulator.client import FakeLLMClient, ResponseCache, ScriptExhaustedError +from pf1e_simulator.client import ( + ClientParams, + Completion, + FakeLLMClient, + ResponseCache, + ScriptExhaustedError, +) from pf1e_simulator.combat import Action, Observation from pf1e_simulator.nl_policy import ( NLPolicy, NLPolicyError, + NLPolicyStats, fallback_default, parse_actions, ) + +if TYPE_CHECKING: + from collections.abc import Sequence from pf1e_simulator.prompt import PromptPack from pf1e_simulator.prompt_builder import PromptBuilder @@ -51,8 +62,26 @@ def _make_client(script: list[str], *, builder: PromptBuilder, **kwargs) -> Fake ) -def _make_policy(client: FakeLLMClient, builder: PromptBuilder) -> NLPolicy: - return NLPolicy(client=client, builder=builder) +def _make_policy( + client: FakeLLMClient, builder: PromptBuilder, *, max_repairs: int = 1 +) -> NLPolicy: + return NLPolicy(client=client, builder=builder, max_repairs=max_repairs) + + +class _RecordingClient(FakeLLMClient): + """Scripted client that records every prompt sent to the model. + + The repair-loop tests assert on the accumulated error trail, so the + prompts themselves must be observable. + """ + + def __init__(self, script: Sequence[str], *, prompt_version: str) -> None: + super().__init__(script=script, prompt_version=prompt_version) + self.prompts: list[str] = [] + + def _call_once(self, prompt: str, *, params: ClientParams) -> Completion: + self.prompts.append(prompt) + return super()._call_once(prompt, params=params) @pytest.fixture @@ -278,3 +307,108 @@ def test_nl_policy_shared_cache_hit() -> None: # Second call was a cache hit (no ScriptExhaustedError) assert client2.stats.cache_hits == 1 assert client1.stats.cache_misses == 1 + + +# ---- P5.2 repair loop -------------------------------------------------------- + +def test_nl_policy_stats_defaults_to_zero() -> None: + """A fresh NLPolicy reports zero calls until the first decision.""" + builder = _builder() + client = _make_client(['[{"kind": "wait"}]'], builder=builder) + policy = _make_policy(client, builder) + assert policy.stats == NLPolicyStats() + assert policy.stats.calls == 0 + + +def test_nl_policy_max_repairs_zero_falls_back_immediately() -> None: + """max_repairs=0 disables repair: first bad reply triggers fallback.""" + builder = _builder() + client = _make_client(["bad one"], builder=builder) + policy = _make_policy(client, builder, max_repairs=0) + + actions = policy(_obs()) + assert actions == (Action(kind="total_defense"),) + assert policy.stats.calls == 1 + assert policy.stats.repairs_attempted == 0 + assert policy.stats.repairs_succeeded == 0 + assert policy.stats.fallbacks == 1 + + +def test_nl_policy_max_repairs_two_repairs_then_succeeds() -> None: + """Two invalid replies are repaired, then the valid reply is parsed.""" + builder = _builder() + script = ["bad one", "bad two", '[{"kind": "move", "destination": [2, 3]}]'] + client = _make_client(script, builder=builder) + policy = _make_policy(client, builder, max_repairs=2) + + actions = policy(_obs()) + assert actions == (Action(kind="move", destination=(2, 3)),) + assert policy.stats.calls == 3 + assert policy.stats.repairs_attempted == 2 + assert policy.stats.repairs_succeeded == 1 + assert policy.stats.fallbacks == 0 + + +def test_nl_policy_max_repairs_two_then_falls_back() -> None: + """Past max_repairs the policy stops repairing and falls back.""" + builder = _builder() + script = ["bad one", "bad two", "bad three"] + client = _make_client(script, builder=builder) + policy = _make_policy(client, builder, max_repairs=2) + + actions = policy(_obs()) + assert actions == (Action(kind="total_defense"),) + assert policy.stats.calls == 3 + assert policy.stats.repairs_attempted == 2 + assert policy.stats.repairs_succeeded == 0 + assert policy.stats.fallbacks == 1 + + +def test_nl_policy_repair_trail_accumulates() -> None: + """Each retry prompt carries the full accumulated error trail.""" + builder = _builder() + script = ["bad one", "bad two", '[{"kind": "wait"}]'] + client = _RecordingClient( + script=script, + prompt_version=builder.prompt_version("nl_default"), + ) + policy = _make_policy(client, builder, max_repairs=2) + + actions = policy(_obs()) + assert actions == (Action(kind="wait"),) + + # First call: bare prompt with no trail. + assert len(client.prompts) == 3 + assert "-- previous reply invalid" not in client.prompts[0] + # First retry: trail carries attempt 1 only. + assert "attempt 1" in client.prompts[1] + assert "attempt 2" not in client.prompts[1] + assert "Return a valid JSON action sequence." in client.prompts[1] + # Second retry: trail carries attempts 1 AND 2, in order. + attempt1 = client.prompts[2].index("-- previous reply invalid (attempt 1) --") + attempt2 = client.prompts[2].index("-- previous reply invalid (attempt 2) --") + assert attempt1 < attempt2 + # The base prompt is preserved at the head of every retry. + assert client.prompts[1].startswith(client.prompts[0]) + + +def test_nl_policy_max_repairs_negative_rejected() -> None: + """A negative max_repairs is a configuration error, not a fallback.""" + builder = _builder() + client = _make_client(['[{"kind": "wait"}]'], builder=builder) + with pytest.raises(ValueError, match=r"max_repairs must be >= 0, got -1"): + _make_policy(client, builder, max_repairs=-1) + + +def test_nl_policy_stats_after_direct_success() -> None: + """A clean first-try success leaves the repair counters untouched.""" + builder = _builder() + client = _make_client(['[{"kind": "wait"}]'], builder=builder) + policy = _make_policy(client, builder) + + actions = policy(_obs()) + assert actions == (Action(kind="wait"),) + assert policy.stats.calls == 1 + assert policy.stats.repairs_attempted == 0 + assert policy.stats.repairs_succeeded == 0 + assert policy.stats.fallbacks == 0 diff --git a/tests/test_phase2_prompt.py b/tests/test_phase2_prompt.py index 3c0d108..76c39ea 100644 --- a/tests/test_phase2_prompt.py +++ b/tests/test_phase2_prompt.py @@ -24,6 +24,7 @@ from pf1e_simulator.config import ( llm_api_key, ) from pf1e_simulator.prompt import ( + FewShotExample, PromptPack, PromptPackError, clear_pack_cache, @@ -31,6 +32,7 @@ from pf1e_simulator.prompt import ( ) FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack" +_FIXTURE_PACK_FS = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" _FIXTURE_VERSION = "0.1.0-test" _FIXTURE_PROMPT_NAME = "nl_default" @@ -43,6 +45,12 @@ def _write_pack(root: Path, *, version: str, prompts: dict[str, str]) -> None: (root / f"{name}.md").write_text(text, encoding="utf-8") +def _write_examples(root: Path, examples: dict[str, str]) -> None: + """Write .examples.json files into an existing pack directory.""" + for name, text in examples.items(): + (root / f"{name}.examples.json").write_text(text, encoding="utf-8") + + # ── config.py ────────────────────────────────────────────────────────────── @@ -169,3 +177,145 @@ def test_load_returns_prompt_pack_instance() -> None: assert isinstance(pack, PromptPack) assert pack.version == _FIXTURE_VERSION + +# ── prompt.py — few-shot examples (P5.2) ──────────────────────────────────── + + +def test_examples_absent_returns_empty() -> None: + """A prompt without an examples file exposes an empty example tuple.""" + pack = load_prompt_pack(FIXTURE_PACK) + assert pack.examples(_FIXTURE_PROMPT_NAME) == () + + +def test_few_shot_fixture_pack_loads_examples() -> None: + """prompt_pack_fs ships one teaching pair, parsed into FewShotExample.""" + pack = load_prompt_pack(_FIXTURE_PACK_FS) + examples = pack.examples(_FIXTURE_PROMPT_NAME) + assert len(examples) == 1 + example = examples[0] + assert isinstance(example, FewShotExample) + assert example.observation == { + "round": 1, + "combatant": "goblin", + "enemy": "orc", + "distance_ft": 10, + } + assert example.actions == ({"kind": "attack", "target_id": "orc"},) + + +def test_prompt_version_includes_examples_content() -> None: + """With an examples file, the content hash covers it (cache key, D6).""" + plain = load_prompt_pack(FIXTURE_PACK) + with_examples = load_prompt_pack(_FIXTURE_PACK_FS) + # Same prompt text, same pack version: the digest must differ because the + # few-shot file is hashed alongside the prompt. + assert plain.get(_FIXTURE_PROMPT_NAME) == with_examples.get(_FIXTURE_PROMPT_NAME) + assert plain.prompt_version(_FIXTURE_PROMPT_NAME) != with_examples.prompt_version( + _FIXTURE_PROMPT_NAME + ) + + +def test_prompt_version_byte_identical_without_examples(tmp_path: Path) -> None: + """No examples file → digest is byte-identical to the pre-few-shot scheme.""" + root = tmp_path / "plain" + _write_pack(root, version=_FIXTURE_VERSION, prompts={_FIXTURE_PROMPT_NAME: "x"}) + pack = load_prompt_pack(root) + digest = pack.prompt_version(_FIXTURE_PROMPT_NAME).split(":")[-1] + assert len(digest) == 12 + + +def test_examples_invalid_json_raises(tmp_path: Path) -> None: + """A malformed examples file is a pack error, not a silent skip.""" + root = tmp_path / "badjson" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples(root, {_FIXTURE_PROMPT_NAME: "not json"}) + with pytest.raises(PromptPackError, match="is not valid JSON"): + load_prompt_pack(root) + + +def test_examples_not_an_array_raises(tmp_path: Path) -> None: + """The examples file must be a JSON array at the top level.""" + root = tmp_path / "notarray" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples(root, {_FIXTURE_PROMPT_NAME: '{"observation": {}}'}) + with pytest.raises(PromptPackError, match="must be a JSON array of examples"): + load_prompt_pack(root) + + +def test_examples_item_not_object_raises(tmp_path: Path) -> None: + """Each array entry must be an object with observation and actions.""" + root = tmp_path / "itemnotobject" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples(root, {_FIXTURE_PROMPT_NAME: "[42]"}) + with pytest.raises( + PromptPackError, + match="must be an object with 'observation' and 'actions'", + ): + load_prompt_pack(root) + + +def test_examples_unknown_key_raises(tmp_path: Path) -> None: + """Unexpected keys inside an example are rejected (strict validation).""" + root = tmp_path / "unknownkey" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples( + root, + {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": [], "hint": 1}]'}, + ) + with pytest.raises(PromptPackError, match="unknown key\\(s\\): hint"): + load_prompt_pack(root) + + +def test_examples_observation_not_object_raises(tmp_path: Path) -> None: + """'observation' must be a JSON object.""" + root = tmp_path / "badobs" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples( + root, + {_FIXTURE_PROMPT_NAME: '[{"observation": "goblin", "actions": []}]'}, + ) + with pytest.raises(PromptPackError, match="'observation' must be a JSON object"): + load_prompt_pack(root) + + +def test_examples_actions_not_array_raises(tmp_path: Path) -> None: + """'actions' must be a JSON array.""" + root = tmp_path / "badactions" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples( + root, + {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": {"kind": "wait"}}]'}, + ) + with pytest.raises(PromptPackError, match="'actions' must be a JSON array"): + load_prompt_pack(root) + + +def test_examples_action_entry_not_object_raises(tmp_path: Path) -> None: + """Each action inside 'actions' must be an object.""" + root = tmp_path / "badactionentry" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples( + root, + {_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": ["jump"]}]'}, + ) + with pytest.raises(PromptPackError, match="must contain only JSON objects"): + load_prompt_pack(root) + + +def test_examples_empty_array_accepted(tmp_path: Path) -> None: + """An empty examples array is valid and yields no teaching pairs.""" + root = tmp_path / "emptyexamples" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples(root, {_FIXTURE_PROMPT_NAME: "[]"}) + pack = load_prompt_pack(root) + assert pack.examples(_FIXTURE_PROMPT_NAME) == () + + +def test_orphan_examples_file_raises(tmp_path: Path) -> None: + """An .examples.json without a matching .md is a pack error.""" + root = tmp_path / "orphan" + _write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"}) + _write_examples(root, {"stray": "[]"}) + with pytest.raises(PromptPackError, match="has no matching 'stray\\.md' prompt"): + load_prompt_pack(root) + diff --git a/tests/test_phase2_prompt_builder.py b/tests/test_phase2_prompt_builder.py index 0b82090..26e42fe 100644 --- a/tests/test_phase2_prompt_builder.py +++ b/tests/test_phase2_prompt_builder.py @@ -13,12 +13,14 @@ from pf1e_simulator.nl_policy import get_args as _get_args from pf1e_simulator.prompt import PromptPack from pf1e_simulator.prompt_builder import ( PromptBuilder, + _render_examples, _supported_kinds, action_schema, render_candidates, ) _FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack" +_FIXTURE_PACK_FS = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs" _FIXTURE_VERSION = "0.1.0-test" _FIXTURE_PROMPT_NAME = "nl_default" @@ -221,3 +223,75 @@ def test_action_schema_cross_pin_with_nl_policy_kindvalue() -> None: """The kind enum in the schema must match nl_policy.KindValue exactly.""" schema = action_schema() assert schema["properties"]["kind"]["enum"] == list(_get_args(KindValue)) + + +# ---- few-shot examples (P5.2) ---------------------------------------------- + +_FS_PACK = PromptPack.load(_FIXTURE_PACK_FS) + + +def _fs_builder() -> PromptBuilder: + return PromptBuilder(pack=_FS_PACK) + + +def test_render_examples_block_format() -> None: + """Few-shot pairs render as numbered observation-to-response blocks.""" + rendered = _render_examples((_FS_PACK.examples(_FIXTURE_PROMPT_NAME)[0],)) + assert rendered == ( + "## few-shot examples\n" + "### example 1\n" + "observation:\n" + '{"combatant": "goblin", "distance_ft": 10, "enemy": "orc", "round": 1}\n' + "\n" + "response:\n" + '[{"kind": "attack", "target_id": "orc"}]' + ) + + +def test_build_without_examples_unchanged() -> None: + """Packs without examples keep the pre-few-shot bit-for-bit section order.""" + plain = _builder().build(_obs()) + assert "## few-shot examples" not in plain + # Template, then observation, then candidates, then schema — in order. + sections = [ + "## observation\n", + "## plausible actions\n", + "## action schema\n", + ] + positions = [plain.index(section) for section in sections] + assert positions == sorted(positions) + # The template comes first, unmodified. + assert plain.startswith("You are the tactical director") + + +def test_build_with_examples_inserts_between_template_and_observation() -> None: + """The few-shot section sits between the template and the observation.""" + prompt = _fs_builder().build(_obs()) + assert "## few-shot examples\n" in prompt + assert prompt.index("## few-shot examples\n") < prompt.index("## observation\n") + # Template still leads the prompt. + assert prompt.startswith("You are the tactical director") + # Observation/candidates/schema keep their relative order. + assert prompt.index("## observation\n") < prompt.index("## plausible actions\n") + assert prompt.index("## plausible actions\n") < prompt.index("## action schema\n") + + +def test_build_with_examples_includes_example_content() -> None: + """The rendered example content is embedded verbatim (sorted keys).""" + prompt = _fs_builder().build(_obs()) + assert '{"combatant": "goblin", "distance_ft": 10, "enemy": "orc", "round": 1}' in prompt + assert '[{"kind": "attack", "target_id": "orc"}]' in prompt + assert "### example 1" in prompt + + +def test_empty_examples_array_no_section(tmp_path: Path) -> None: + """An empty examples array yields no few-shot section (not an error).""" + root = tmp_path / "emptyfs" + root.mkdir(parents=True, exist_ok=True) + (root / "version.txt").write_text("1.0.0", encoding="utf-8") + (root / "nl_default.md").write_text("T", encoding="utf-8") + (root / "nl_default.examples.json").write_text("[]", encoding="utf-8") + pack = PromptPack.load(root) + assert pack.examples("nl_default") == () + prompt = PromptBuilder(pack=pack).build(_obs()) + assert "## few-shot examples" not in prompt