322 lines
13 KiB
Python
322 lines
13 KiB
Python
"""Phase-2 config and external prompt pack (docs/phase-2 P1a, sections 14-15).
|
|
|
|
``config.py`` centralizes every phase-2 path/env read (no hard-coded data
|
|
paths - section 15). ``prompt.py`` loads the EXTERNAL, versioned prompt pack
|
|
(section 14): prompts never live in this repo; the pack carries its own
|
|
version and each prompt has an individual content hash, so ``prompt_version``
|
|
can enter the response-cache composite key (D6). The minimal pack under
|
|
``tests/fixtures/prompt_pack/`` is the local fixture so CI never depends on
|
|
the external prompt repository, and missing/invalid packs raise a clear
|
|
``PromptPackError`` instead of falling back silently.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from pf1e_simulator.config import (
|
|
ENV_LLM_API_KEY,
|
|
ENV_PROMPT_PACK_DIR,
|
|
env_path,
|
|
env_str,
|
|
llm_api_key,
|
|
)
|
|
from pf1e_simulator.prompt import (
|
|
FewShotExample,
|
|
PromptPack,
|
|
PromptPackError,
|
|
clear_pack_cache,
|
|
load_prompt_pack,
|
|
)
|
|
|
|
FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack"
|
|
_FIXTURE_PACK_FS = Path(__file__).resolve().parent / "fixtures" / "prompt_pack_fs"
|
|
_FIXTURE_VERSION = "0.1.0-test"
|
|
_FIXTURE_PROMPT_NAME = "nl_default"
|
|
|
|
|
|
def _write_pack(root: Path, *, version: str, prompts: dict[str, str]) -> None:
|
|
"""Write a minimal prompt pack directory (version.txt + <name>.md)."""
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
(root / "version.txt").write_text(version, encoding="utf-8")
|
|
for name, text in prompts.items():
|
|
(root / f"{name}.md").write_text(text, encoding="utf-8")
|
|
|
|
|
|
def _write_examples(root: Path, examples: dict[str, str]) -> None:
|
|
"""Write <name>.examples.json files into an existing pack directory."""
|
|
for name, text in examples.items():
|
|
(root / f"{name}.examples.json").write_text(text, encoding="utf-8")
|
|
|
|
|
|
# ── config.py ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_env_str_and_path_round_trip(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""env_str/env_path read the environment and respect deletes."""
|
|
monkeypatch.setenv("PF1E_SIM_TEST_STR", "abc")
|
|
monkeypatch.setenv("PF1E_SIM_TEST_PATH", str(tmp_path / "x"))
|
|
assert env_str("PF1E_SIM_TEST_STR") == "abc"
|
|
assert env_str("PF1E_SIM_TEST_UNSET") is None
|
|
assert env_str("PF1E_SIM_TEST_UNSET", default="d") == "d"
|
|
assert env_path("PF1E_SIM_TEST_PATH") == tmp_path / "x"
|
|
assert env_path("PF1E_SIM_TEST_UNSET") is None
|
|
monkeypatch.delenv("PF1E_SIM_TEST_STR")
|
|
assert env_str("PF1E_SIM_TEST_STR") is None
|
|
|
|
|
|
def test_llm_api_key_reads_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""llm_api_key surfaces the dedicated env var (None when unset)."""
|
|
assert llm_api_key() is None
|
|
monkeypatch.setenv(ENV_LLM_API_KEY, "sk-test")
|
|
assert llm_api_key() == "sk-test"
|
|
|
|
|
|
# ── prompt.py — loading and validation ─────────────────────────────────────
|
|
|
|
|
|
def test_load_fixture_pack() -> None:
|
|
"""The local fixture loads: version pinned, prompts readable by name."""
|
|
pack = load_prompt_pack(FIXTURE_PACK)
|
|
assert pack.version == _FIXTURE_VERSION
|
|
assert pack.root == FIXTURE_PACK
|
|
text = pack.get(_FIXTURE_PROMPT_NAME)
|
|
assert "tactical director" in text
|
|
# The pack has exactly the fixture prompts (only <name>.md files count).
|
|
assert pack.get("nl_default") == (FIXTURE_PACK / "nl_default.md").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
|
|
def test_prompt_version_stable_and_content_sensitive(tmp_path: Path) -> None:
|
|
"""prompt_version carries pack version + per-prompt content hash (D6)."""
|
|
pack = load_prompt_pack(FIXTURE_PACK)
|
|
a = pack.prompt_version(_FIXTURE_PROMPT_NAME)
|
|
b = pack.prompt_version(_FIXTURE_PROMPT_NAME)
|
|
assert a == b # deterministic within a loaded pack
|
|
assert a.startswith(f"{_FIXTURE_VERSION}:{_FIXTURE_PROMPT_NAME}:")
|
|
|
|
other = tmp_path / "other"
|
|
_write_pack(other, version=_FIXTURE_VERSION, prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
assert load_prompt_pack(other).prompt_version(_FIXTURE_PROMPT_NAME) != a
|
|
|
|
bumped = tmp_path / "bumped"
|
|
_write_pack(bumped, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
assert load_prompt_pack(bumped).prompt_version(_FIXTURE_PROMPT_NAME).startswith(
|
|
"1.0.0:"
|
|
)
|
|
|
|
|
|
def test_missing_prompt_raises_clear_error(tmp_path: Path) -> None:
|
|
"""A missing prompt name fails loudly, naming the expected file."""
|
|
other = tmp_path / "other"
|
|
_write_pack(other, version="1.0.0", prompts={"other": "x"})
|
|
with pytest.raises(PromptPackError, match=r"missing.*nl_default\.md"):
|
|
load_prompt_pack(other).get("nl_default")
|
|
|
|
|
|
def test_missing_pack_dir_raises(tmp_path: Path) -> None:
|
|
"""A nonexistent pack path is a clear error, not a silent fallback."""
|
|
with pytest.raises(PromptPackError, match="not found"):
|
|
load_prompt_pack(tmp_path / "nope")
|
|
|
|
|
|
def test_missing_version_file_raises(tmp_path: Path) -> None:
|
|
"""A pack without version.txt is invalid (traceability, section 14)."""
|
|
root = tmp_path / "noversion"
|
|
_write_pack(root, version="", prompts={})
|
|
(root / "version.txt").unlink()
|
|
with pytest.raises(PromptPackError, match="version\\.txt"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_empty_version_file_raises(tmp_path: Path) -> None:
|
|
"""An empty version.txt is invalid (no untraceable pack)."""
|
|
root = tmp_path / "emptyversion"
|
|
_write_pack(root, version="", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
with pytest.raises(PromptPackError, match="empty"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_unset_pack_dir_raises(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Without PF1E_PROMPT_PACK_DIR, loading raises instead of falling back."""
|
|
monkeypatch.delenv(ENV_PROMPT_PACK_DIR, raising=False)
|
|
with pytest.raises(PromptPackError, match=ENV_PROMPT_PACK_DIR):
|
|
load_prompt_pack()
|
|
|
|
|
|
def test_pack_dir_from_env(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""load_prompt_pack() resolves the pack through the env var (section 15)."""
|
|
monkeypatch.setenv(ENV_PROMPT_PACK_DIR, str(FIXTURE_PACK))
|
|
assert load_prompt_pack().version == _FIXTURE_VERSION
|
|
monkeypatch.delenv(ENV_PROMPT_PACK_DIR)
|
|
|
|
|
|
def test_cache_and_clear(tmp_path: Path) -> None:
|
|
"""Repeated loads share the cached instance; clear_pack_cache drops it."""
|
|
root = tmp_path / "cached"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
first = load_prompt_pack(root)
|
|
assert load_prompt_pack(root) is first
|
|
clear_pack_cache()
|
|
assert load_prompt_pack(root) is not first
|
|
|
|
|
|
def test_load_returns_prompt_pack_instance() -> None:
|
|
"""The loaded object is a PromptPack (typed API for the client layer)."""
|
|
pack = load_prompt_pack(FIXTURE_PACK)
|
|
assert isinstance(pack, PromptPack)
|
|
assert pack.version == _FIXTURE_VERSION
|
|
|
|
|
|
# ── prompt.py — few-shot examples (P5.2) ────────────────────────────────────
|
|
|
|
|
|
def test_examples_absent_returns_empty() -> None:
|
|
"""A prompt without an examples file exposes an empty example tuple."""
|
|
pack = load_prompt_pack(FIXTURE_PACK)
|
|
assert pack.examples(_FIXTURE_PROMPT_NAME) == ()
|
|
|
|
|
|
def test_few_shot_fixture_pack_loads_examples() -> None:
|
|
"""prompt_pack_fs ships one teaching pair, parsed into FewShotExample."""
|
|
pack = load_prompt_pack(_FIXTURE_PACK_FS)
|
|
examples = pack.examples(_FIXTURE_PROMPT_NAME)
|
|
assert len(examples) == 1
|
|
example = examples[0]
|
|
assert isinstance(example, FewShotExample)
|
|
assert example.observation == {
|
|
"round": 1,
|
|
"combatant": "goblin",
|
|
"enemy": "orc",
|
|
"distance_ft": 10,
|
|
}
|
|
assert example.actions == ({"kind": "attack", "target_id": "orc"},)
|
|
|
|
|
|
def test_prompt_version_includes_examples_content() -> None:
|
|
"""With an examples file, the content hash covers it (cache key, D6)."""
|
|
plain = load_prompt_pack(FIXTURE_PACK)
|
|
with_examples = load_prompt_pack(_FIXTURE_PACK_FS)
|
|
# Same prompt text, same pack version: the digest must differ because the
|
|
# few-shot file is hashed alongside the prompt.
|
|
assert plain.get(_FIXTURE_PROMPT_NAME) == with_examples.get(_FIXTURE_PROMPT_NAME)
|
|
assert plain.prompt_version(_FIXTURE_PROMPT_NAME) != with_examples.prompt_version(
|
|
_FIXTURE_PROMPT_NAME
|
|
)
|
|
|
|
|
|
def test_prompt_version_byte_identical_without_examples(tmp_path: Path) -> None:
|
|
"""No examples file → digest is byte-identical to the pre-few-shot scheme."""
|
|
root = tmp_path / "plain"
|
|
_write_pack(root, version=_FIXTURE_VERSION, prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
pack = load_prompt_pack(root)
|
|
digest = pack.prompt_version(_FIXTURE_PROMPT_NAME).split(":")[-1]
|
|
assert len(digest) == 12
|
|
|
|
|
|
def test_examples_invalid_json_raises(tmp_path: Path) -> None:
|
|
"""A malformed examples file is a pack error, not a silent skip."""
|
|
root = tmp_path / "badjson"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(root, {_FIXTURE_PROMPT_NAME: "not json"})
|
|
with pytest.raises(PromptPackError, match="is not valid JSON"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_not_an_array_raises(tmp_path: Path) -> None:
|
|
"""The examples file must be a JSON array at the top level."""
|
|
root = tmp_path / "notarray"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(root, {_FIXTURE_PROMPT_NAME: '{"observation": {}}'})
|
|
with pytest.raises(PromptPackError, match="must be a JSON array of examples"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_item_not_object_raises(tmp_path: Path) -> None:
|
|
"""Each array entry must be an object with observation and actions."""
|
|
root = tmp_path / "itemnotobject"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(root, {_FIXTURE_PROMPT_NAME: "[42]"})
|
|
with pytest.raises(
|
|
PromptPackError,
|
|
match="must be an object with 'observation' and 'actions'",
|
|
):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_unknown_key_raises(tmp_path: Path) -> None:
|
|
"""Unexpected keys inside an example are rejected (strict validation)."""
|
|
root = tmp_path / "unknownkey"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(
|
|
root,
|
|
{_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": [], "hint": 1}]'},
|
|
)
|
|
with pytest.raises(PromptPackError, match="unknown key\\(s\\): hint"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_observation_not_object_raises(tmp_path: Path) -> None:
|
|
"""'observation' must be a JSON object."""
|
|
root = tmp_path / "badobs"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(
|
|
root,
|
|
{_FIXTURE_PROMPT_NAME: '[{"observation": "goblin", "actions": []}]'},
|
|
)
|
|
with pytest.raises(PromptPackError, match="'observation' must be a JSON object"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_actions_not_array_raises(tmp_path: Path) -> None:
|
|
"""'actions' must be a JSON array."""
|
|
root = tmp_path / "badactions"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(
|
|
root,
|
|
{_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": {"kind": "wait"}}]'},
|
|
)
|
|
with pytest.raises(PromptPackError, match="'actions' must be a JSON array"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_action_entry_not_object_raises(tmp_path: Path) -> None:
|
|
"""Each action inside 'actions' must be an object."""
|
|
root = tmp_path / "badactionentry"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(
|
|
root,
|
|
{_FIXTURE_PROMPT_NAME: '[{"observation": {}, "actions": ["jump"]}]'},
|
|
)
|
|
with pytest.raises(PromptPackError, match="must contain only JSON objects"):
|
|
load_prompt_pack(root)
|
|
|
|
|
|
def test_examples_empty_array_accepted(tmp_path: Path) -> None:
|
|
"""An empty examples array is valid and yields no teaching pairs."""
|
|
root = tmp_path / "emptyexamples"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(root, {_FIXTURE_PROMPT_NAME: "[]"})
|
|
pack = load_prompt_pack(root)
|
|
assert pack.examples(_FIXTURE_PROMPT_NAME) == ()
|
|
|
|
|
|
def test_orphan_examples_file_raises(tmp_path: Path) -> None:
|
|
"""An .examples.json without a matching <name>.md is a pack error."""
|
|
root = tmp_path / "orphan"
|
|
_write_pack(root, version="1.0.0", prompts={_FIXTURE_PROMPT_NAME: "x"})
|
|
_write_examples(root, {"stray": "[]"})
|
|
with pytest.raises(PromptPackError, match="has no matching 'stray\\.md' prompt"):
|
|
load_prompt_pack(root)
|
|
|