415 lines
14 KiB
Python
415 lines
14 KiB
Python
"""Tests for the natural-language policy (P1.3)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from pf1e_simulator.client import (
|
|
ClientParams,
|
|
Completion,
|
|
FakeLLMClient,
|
|
ResponseCache,
|
|
ScriptExhaustedError,
|
|
)
|
|
from pf1e_simulator.combat import Action, Observation
|
|
from pf1e_simulator.nl_policy import (
|
|
NLPolicy,
|
|
NLPolicyError,
|
|
NLPolicyStats,
|
|
fallback_default,
|
|
parse_actions,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Sequence
|
|
from pf1e_simulator.prompt import PromptPack
|
|
from pf1e_simulator.prompt_builder import PromptBuilder
|
|
|
|
_FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack"
|
|
|
|
|
|
def _obs() -> Observation:
|
|
"""Minimal valid observation for deterministic tests."""
|
|
return Observation(
|
|
round=1,
|
|
acting_combatant=0,
|
|
combatants=(
|
|
{"id": "goblin", "side": "players", "active": True, "pos": (1, 1)},
|
|
),
|
|
pairs=(),
|
|
terrain=((None,),),
|
|
elevation=((0,),),
|
|
)
|
|
|
|
|
|
def _pack() -> PromptPack:
|
|
return PromptPack.load(_FIXTURE_PACK)
|
|
|
|
|
|
def _builder() -> PromptBuilder:
|
|
return PromptBuilder(pack=_pack())
|
|
|
|
|
|
def _make_client(script: list[str], *, builder: PromptBuilder, **kwargs) -> FakeLLMClient:
|
|
"""Create a FakeLLMClient with the correct prompt_version from the builder."""
|
|
return FakeLLMClient(
|
|
script=script,
|
|
prompt_version=builder.prompt_version("nl_default"),
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def _make_policy(
|
|
client: FakeLLMClient, builder: PromptBuilder, *, max_repairs: int = 1
|
|
) -> NLPolicy:
|
|
return NLPolicy(client=client, builder=builder, max_repairs=max_repairs)
|
|
|
|
|
|
class _RecordingClient(FakeLLMClient):
|
|
"""Scripted client that records every prompt sent to the model.
|
|
|
|
The repair-loop tests assert on the accumulated error trail, so the
|
|
prompts themselves must be observable.
|
|
"""
|
|
|
|
def __init__(self, script: Sequence[str], *, prompt_version: str) -> None:
|
|
super().__init__(script=script, prompt_version=prompt_version)
|
|
self.prompts: list[str] = []
|
|
|
|
def _call_once(self, prompt: str, *, params: ClientParams) -> Completion:
|
|
self.prompts.append(prompt)
|
|
return super()._call_once(prompt, params=params)
|
|
|
|
|
|
@pytest.fixture
|
|
def builder() -> PromptBuilder:
|
|
"""A builder over the fixture pack, shared by the NLPolicy integration tests."""
|
|
return _builder()
|
|
|
|
|
|
# ---- parse_actions tests -----------------------------------------------------
|
|
|
|
def test_parse_actions_ok() -> None:
|
|
reply = '[{"kind": "move", "destination": [2, 3]}]'
|
|
actions = parse_actions(reply)
|
|
assert actions == (Action(kind="move", destination=(2, 3)),)
|
|
|
|
|
|
def test_parse_actions_empty_array() -> None:
|
|
assert parse_actions("[]") == ()
|
|
|
|
|
|
def test_parse_actions_invalid_json() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"reply is not valid JSON"):
|
|
parse_actions("not json")
|
|
|
|
|
|
def test_parse_actions_not_a_list() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"reply must be a JSON array of action objects"):
|
|
parse_actions('{"kind": "attack"}')
|
|
|
|
|
|
def test_parse_actions_unknown_field() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"unknown action field\(s\): bogus"):
|
|
parse_actions('[{"kind": "attack", "bogus": 1}]')
|
|
|
|
|
|
def test_parse_actions_missing_kind() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"missing or unknown 'kind'"):
|
|
parse_actions('[{"target_id": "orc"}]')
|
|
|
|
|
|
def test_parse_actions_unknown_kind() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"missing or unknown 'kind'"):
|
|
parse_actions('[{"kind": "dance"}]')
|
|
|
|
|
|
def test_parse_actions_channel_mode_invalid() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'channel_mode' must be one of: heal, harm"):
|
|
parse_actions('[{"kind": "channel_energy", "channel_mode": "loud"}]')
|
|
|
|
|
|
def test_parse_actions_destination_invalid_not_pair() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'destination' must be a \[row, col\] pair"):
|
|
parse_actions('[{"kind": "move", "destination": [2]}]')
|
|
|
|
|
|
def test_parse_actions_destination_invalid_strings() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'destination' must be a \[row, col\] pair"):
|
|
parse_actions('[{"kind": "move", "destination": ["2", "3"]}]')
|
|
|
|
|
|
def test_parse_actions_aid_mode_invalid() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'aid_mode' must be one of: attack, ac"):
|
|
parse_actions('[{"kind": "aid_another", "aid_mode": "damage"}]')
|
|
|
|
|
|
def test_parse_actions_trick_condition_invalid() -> None:
|
|
with pytest.raises(
|
|
NLPolicyError,
|
|
match=(
|
|
r"'trick_condition' must be one of: dazzled, blinded, "
|
|
r"deafened, entangled, shaken, sickened"
|
|
),
|
|
):
|
|
parse_actions('[{"kind": "cast_spell", "trick_condition": "nauseated"}]')
|
|
|
|
|
|
def test_parse_actions_ready_trigger_invalid() -> None:
|
|
with pytest.raises(
|
|
NLPolicyError,
|
|
match=(
|
|
r"'ready_trigger' must be one of: enemy_enters_reach, "
|
|
r"enemy_casts, enemy_attacks, enemy_hits_ally"
|
|
),
|
|
):
|
|
parse_actions('[{"kind": "ready", "ready_trigger": "enemy_sneezed"}]')
|
|
|
|
|
|
def test_parse_actions_channel_exclude_not_list() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'channel_exclude' must be a list of strings"):
|
|
parse_actions('[{"kind": "channel_energy", "channel_exclude": "orc"}]')
|
|
|
|
|
|
def test_parse_actions_channel_exclude_ok() -> None:
|
|
actions = parse_actions('[{"kind": "channel_energy", "channel_exclude": ["orc"]}]')
|
|
assert actions[0].channel_exclude == ("orc",)
|
|
|
|
|
|
def test_parse_actions_tumble_not_bool() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'tumble' must be a boolean"):
|
|
parse_actions('[{"kind": "attack", "tumble": "yes"}]')
|
|
|
|
|
|
def test_parse_actions_cast_defensively_not_bool() -> None:
|
|
with pytest.raises(NLPolicyError, match=r"'cast_defensively' must be a boolean"):
|
|
parse_actions('[{"kind": "cast_spell", "cast_defensively": 1}]')
|
|
|
|
|
|
def test_parse_actions_metamagic_ok() -> None:
|
|
actions = parse_actions('[{"kind": "cast_spell", "metamagic": ["Empower", "Maximize"]}]')
|
|
assert actions[0].metamagic == ("Empower", "Maximize")
|
|
|
|
|
|
# ---- fallback_default tests --------------------------------------------------
|
|
|
|
def test_fallback_default_prefers_full_attack() -> None:
|
|
menu = (
|
|
Action(kind="total_defense"),
|
|
Action(kind="full_attack", target_id="orc"),
|
|
Action(kind="move"),
|
|
)
|
|
assert fallback_default(menu) == (Action(kind="full_attack", target_id="orc"),)
|
|
|
|
|
|
def test_fallback_default_prefers_attack_over_charge() -> None:
|
|
menu = (
|
|
Action(kind="charge", target_id="orc"),
|
|
Action(kind="attack", target_id="orc"),
|
|
)
|
|
assert fallback_default(menu) == (Action(kind="attack", target_id="orc"),)
|
|
|
|
|
|
def test_fallback_default_prefers_charge_over_move() -> None:
|
|
menu = (
|
|
Action(kind="move", target_id="orc"),
|
|
Action(kind="charge", target_id="orc"),
|
|
)
|
|
assert fallback_default(menu) == (Action(kind="charge", target_id="orc"),)
|
|
|
|
|
|
def test_fallback_default_falls_back_to_first() -> None:
|
|
menu = (Action(kind="total_defense"), Action(kind="delay"))
|
|
assert fallback_default(menu) == (Action(kind="total_defense"),)
|
|
|
|
|
|
def test_fallback_default_empty_menu_returns_wait() -> None:
|
|
assert fallback_default(()) == (Action(kind="wait"),)
|
|
|
|
|
|
# ---- NLPolicy integration tests ---------------------------------------------
|
|
|
|
def test_nl_policy_parse_ok(builder: PromptBuilder) -> None:
|
|
"""Parse OK path with valid JSON from FakeLLMClient."""
|
|
script = ['[{"kind": "move", "destination": [2, 3]}]']
|
|
client = _make_client(script, builder=builder)
|
|
policy = _make_policy(client, builder)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="move", destination=(2, 3)),)
|
|
assert client.stats.calls == 1
|
|
|
|
|
|
def test_nl_policy_retry_then_succeeds() -> None:
|
|
"""One bad reply, then a good one → parsed successfully."""
|
|
builder = _builder()
|
|
script = ["not json", '[{"kind": "move", "destination": [2, 3]}]']
|
|
client = _make_client(script, builder=builder)
|
|
policy = _make_policy(client, builder)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="move", destination=(2, 3)),)
|
|
assert client.stats.calls == 2
|
|
|
|
|
|
def test_nl_policy_retry_then_fallback() -> None:
|
|
"""Two bad replies → fallback used."""
|
|
builder = _builder()
|
|
script = ["not json", "not json"]
|
|
client = _make_client(script, builder=builder)
|
|
policy = _make_policy(client, builder)
|
|
|
|
# Minimal obs menu is (total_defense, delay)
|
|
actions = policy(_obs())
|
|
# fallback_default picks first decisive action, but menu has only defensive
|
|
assert actions == (Action(kind="total_defense"),)
|
|
assert client.stats.calls == 2
|
|
|
|
|
|
def test_nl_policy_prompt_version_mismatch() -> None:
|
|
"""Client default prompt_version vs builder pack version → ValueError."""
|
|
builder = _builder()
|
|
# Use default prompt_version ("fixture:fake:0") which differs from pack
|
|
client = FakeLLMClient(script=['[{"kind": "wait"}]'])
|
|
with pytest.raises(ValueError, match=r"prompt_version"):
|
|
_make_policy(client, builder)
|
|
|
|
|
|
def test_nl_policy_script_exhausted_propagates() -> None:
|
|
"""ScriptExhaustedError from FakeLLMClient propagates (not caught)."""
|
|
builder = _builder()
|
|
client = _make_client([], builder=builder)
|
|
policy = _make_policy(client, builder)
|
|
|
|
with pytest.raises(ScriptExhaustedError):
|
|
policy(_obs())
|
|
|
|
|
|
def test_nl_policy_shared_cache_hit() -> None:
|
|
"""Two clients sharing a ResponseCache: second call hits cache."""
|
|
builder = _builder()
|
|
cache = ResponseCache()
|
|
# First client has one scripted response
|
|
client1 = _make_client(['[{"kind": "wait"}]'], builder=builder, cache=cache)
|
|
# Second client has NO script; relies on cache hit
|
|
client2 = _make_client([], builder=builder, cache=cache)
|
|
policy1 = _make_policy(client1, builder)
|
|
policy2 = _make_policy(client2, builder)
|
|
|
|
obs = _obs()
|
|
actions1 = policy1(obs)
|
|
actions2 = policy2(obs)
|
|
|
|
assert actions2 == actions1 == (Action(kind="wait"),)
|
|
# Second call was a cache hit (no ScriptExhaustedError)
|
|
assert client2.stats.cache_hits == 1
|
|
assert client1.stats.cache_misses == 1
|
|
|
|
|
|
# ---- P5.2 repair loop --------------------------------------------------------
|
|
|
|
def test_nl_policy_stats_defaults_to_zero() -> None:
|
|
"""A fresh NLPolicy reports zero calls until the first decision."""
|
|
builder = _builder()
|
|
client = _make_client(['[{"kind": "wait"}]'], builder=builder)
|
|
policy = _make_policy(client, builder)
|
|
assert policy.stats == NLPolicyStats()
|
|
assert policy.stats.calls == 0
|
|
|
|
|
|
def test_nl_policy_max_repairs_zero_falls_back_immediately() -> None:
|
|
"""max_repairs=0 disables repair: first bad reply triggers fallback."""
|
|
builder = _builder()
|
|
client = _make_client(["bad one"], builder=builder)
|
|
policy = _make_policy(client, builder, max_repairs=0)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="total_defense"),)
|
|
assert policy.stats.calls == 1
|
|
assert policy.stats.repairs_attempted == 0
|
|
assert policy.stats.repairs_succeeded == 0
|
|
assert policy.stats.fallbacks == 1
|
|
|
|
|
|
def test_nl_policy_max_repairs_two_repairs_then_succeeds() -> None:
|
|
"""Two invalid replies are repaired, then the valid reply is parsed."""
|
|
builder = _builder()
|
|
script = ["bad one", "bad two", '[{"kind": "move", "destination": [2, 3]}]']
|
|
client = _make_client(script, builder=builder)
|
|
policy = _make_policy(client, builder, max_repairs=2)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="move", destination=(2, 3)),)
|
|
assert policy.stats.calls == 3
|
|
assert policy.stats.repairs_attempted == 2
|
|
assert policy.stats.repairs_succeeded == 1
|
|
assert policy.stats.fallbacks == 0
|
|
|
|
|
|
def test_nl_policy_max_repairs_two_then_falls_back() -> None:
|
|
"""Past max_repairs the policy stops repairing and falls back."""
|
|
builder = _builder()
|
|
script = ["bad one", "bad two", "bad three"]
|
|
client = _make_client(script, builder=builder)
|
|
policy = _make_policy(client, builder, max_repairs=2)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="total_defense"),)
|
|
assert policy.stats.calls == 3
|
|
assert policy.stats.repairs_attempted == 2
|
|
assert policy.stats.repairs_succeeded == 0
|
|
assert policy.stats.fallbacks == 1
|
|
|
|
|
|
def test_nl_policy_repair_trail_accumulates() -> None:
|
|
"""Each retry prompt carries the full accumulated error trail."""
|
|
builder = _builder()
|
|
script = ["bad one", "bad two", '[{"kind": "wait"}]']
|
|
client = _RecordingClient(
|
|
script=script,
|
|
prompt_version=builder.prompt_version("nl_default"),
|
|
)
|
|
policy = _make_policy(client, builder, max_repairs=2)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="wait"),)
|
|
|
|
# First call: bare prompt with no trail.
|
|
assert len(client.prompts) == 3
|
|
assert "-- previous reply invalid" not in client.prompts[0]
|
|
# First retry: trail carries attempt 1 only.
|
|
assert "attempt 1" in client.prompts[1]
|
|
assert "attempt 2" not in client.prompts[1]
|
|
assert "Return a valid JSON action sequence." in client.prompts[1]
|
|
# Second retry: trail carries attempts 1 AND 2, in order.
|
|
attempt1 = client.prompts[2].index("-- previous reply invalid (attempt 1) --")
|
|
attempt2 = client.prompts[2].index("-- previous reply invalid (attempt 2) --")
|
|
assert attempt1 < attempt2
|
|
# The base prompt is preserved at the head of every retry.
|
|
assert client.prompts[1].startswith(client.prompts[0])
|
|
|
|
|
|
def test_nl_policy_max_repairs_negative_rejected() -> None:
|
|
"""A negative max_repairs is a configuration error, not a fallback."""
|
|
builder = _builder()
|
|
client = _make_client(['[{"kind": "wait"}]'], builder=builder)
|
|
with pytest.raises(ValueError, match=r"max_repairs must be >= 0, got -1"):
|
|
_make_policy(client, builder, max_repairs=-1)
|
|
|
|
|
|
def test_nl_policy_stats_after_direct_success() -> None:
|
|
"""A clean first-try success leaves the repair counters untouched."""
|
|
builder = _builder()
|
|
client = _make_client(['[{"kind": "wait"}]'], builder=builder)
|
|
policy = _make_policy(client, builder)
|
|
|
|
actions = policy(_obs())
|
|
assert actions == (Action(kind="wait"),)
|
|
assert policy.stats.calls == 1
|
|
assert policy.stats.repairs_attempted == 0
|
|
assert policy.stats.repairs_succeeded == 0
|
|
assert policy.stats.fallbacks == 0
|