"""Tests for the natural-language policy (P1.3).""" from __future__ import annotations from pathlib import Path from typing import TYPE_CHECKING import pytest from pf1e_simulator.client import ( ClientParams, Completion, FakeLLMClient, ResponseCache, ScriptExhaustedError, ) from pf1e_simulator.combat import Action, Observation from pf1e_simulator.nl_policy import ( NLPolicy, NLPolicyError, NLPolicyStats, fallback_default, parse_actions, ) if TYPE_CHECKING: from collections.abc import Sequence from pf1e_simulator.prompt import PromptPack from pf1e_simulator.prompt_builder import PromptBuilder _FIXTURE_PACK = Path(__file__).resolve().parent / "fixtures" / "prompt_pack" def _obs() -> Observation: """Minimal valid observation for deterministic tests.""" return Observation( round=1, acting_combatant=0, combatants=( {"id": "goblin", "side": "players", "active": True, "pos": (1, 1)}, ), pairs=(), terrain=((None,),), elevation=((0,),), ) def _pack() -> PromptPack: return PromptPack.load(_FIXTURE_PACK) def _builder() -> PromptBuilder: return PromptBuilder(pack=_pack()) def _make_client(script: list[str], *, builder: PromptBuilder, **kwargs) -> FakeLLMClient: """Create a FakeLLMClient with the correct prompt_version from the builder.""" return FakeLLMClient( script=script, prompt_version=builder.prompt_version("nl_default"), **kwargs, ) def _make_policy( client: FakeLLMClient, builder: PromptBuilder, *, max_repairs: int = 1 ) -> NLPolicy: return NLPolicy(client=client, builder=builder, max_repairs=max_repairs) class _RecordingClient(FakeLLMClient): """Scripted client that records every prompt sent to the model. The repair-loop tests assert on the accumulated error trail, so the prompts themselves must be observable. """ def __init__(self, script: Sequence[str], *, prompt_version: str) -> None: super().__init__(script=script, prompt_version=prompt_version) self.prompts: list[str] = [] def _call_once(self, prompt: str, *, params: ClientParams) -> Completion: self.prompts.append(prompt) return super()._call_once(prompt, params=params) @pytest.fixture def builder() -> PromptBuilder: """A builder over the fixture pack, shared by the NLPolicy integration tests.""" return _builder() # ---- parse_actions tests ----------------------------------------------------- def test_parse_actions_ok() -> None: reply = '[{"kind": "move", "destination": [2, 3]}]' actions = parse_actions(reply) assert actions == (Action(kind="move", destination=(2, 3)),) def test_parse_actions_empty_array() -> None: assert parse_actions("[]") == () def test_parse_actions_invalid_json() -> None: with pytest.raises(NLPolicyError, match=r"reply is not valid JSON"): parse_actions("not json") def test_parse_actions_not_a_list() -> None: with pytest.raises(NLPolicyError, match=r"reply must be a JSON array of action objects"): parse_actions('{"kind": "attack"}') def test_parse_actions_unknown_field() -> None: with pytest.raises(NLPolicyError, match=r"unknown action field\(s\): bogus"): parse_actions('[{"kind": "attack", "bogus": 1}]') def test_parse_actions_missing_kind() -> None: with pytest.raises(NLPolicyError, match=r"missing or unknown 'kind'"): parse_actions('[{"target_id": "orc"}]') def test_parse_actions_unknown_kind() -> None: with pytest.raises(NLPolicyError, match=r"missing or unknown 'kind'"): parse_actions('[{"kind": "dance"}]') def test_parse_actions_channel_mode_invalid() -> None: with pytest.raises(NLPolicyError, match=r"'channel_mode' must be one of: heal, harm"): parse_actions('[{"kind": "channel_energy", "channel_mode": "loud"}]') def test_parse_actions_destination_invalid_not_pair() -> None: with pytest.raises(NLPolicyError, match=r"'destination' must be a \[row, col\] pair"): parse_actions('[{"kind": "move", "destination": [2]}]') def test_parse_actions_destination_invalid_strings() -> None: with pytest.raises(NLPolicyError, match=r"'destination' must be a \[row, col\] pair"): parse_actions('[{"kind": "move", "destination": ["2", "3"]}]') def test_parse_actions_aid_mode_invalid() -> None: with pytest.raises(NLPolicyError, match=r"'aid_mode' must be one of: attack, ac"): parse_actions('[{"kind": "aid_another", "aid_mode": "damage"}]') def test_parse_actions_trick_condition_invalid() -> None: with pytest.raises( NLPolicyError, match=( r"'trick_condition' must be one of: dazzled, blinded, " r"deafened, entangled, shaken, sickened" ), ): parse_actions('[{"kind": "cast_spell", "trick_condition": "nauseated"}]') def test_parse_actions_ready_trigger_invalid() -> None: with pytest.raises( NLPolicyError, match=( r"'ready_trigger' must be one of: enemy_enters_reach, " r"enemy_casts, enemy_attacks, enemy_hits_ally" ), ): parse_actions('[{"kind": "ready", "ready_trigger": "enemy_sneezed"}]') def test_parse_actions_channel_exclude_not_list() -> None: with pytest.raises(NLPolicyError, match=r"'channel_exclude' must be a list of strings"): parse_actions('[{"kind": "channel_energy", "channel_exclude": "orc"}]') def test_parse_actions_channel_exclude_ok() -> None: actions = parse_actions('[{"kind": "channel_energy", "channel_exclude": ["orc"]}]') assert actions[0].channel_exclude == ("orc",) def test_parse_actions_tumble_not_bool() -> None: with pytest.raises(NLPolicyError, match=r"'tumble' must be a boolean"): parse_actions('[{"kind": "attack", "tumble": "yes"}]') def test_parse_actions_cast_defensively_not_bool() -> None: with pytest.raises(NLPolicyError, match=r"'cast_defensively' must be a boolean"): parse_actions('[{"kind": "cast_spell", "cast_defensively": 1}]') def test_parse_actions_metamagic_ok() -> None: actions = parse_actions('[{"kind": "cast_spell", "metamagic": ["Empower", "Maximize"]}]') assert actions[0].metamagic == ("Empower", "Maximize") # ---- fallback_default tests -------------------------------------------------- def test_fallback_default_prefers_full_attack() -> None: menu = ( Action(kind="total_defense"), Action(kind="full_attack", target_id="orc"), Action(kind="move"), ) assert fallback_default(menu) == (Action(kind="full_attack", target_id="orc"),) def test_fallback_default_prefers_attack_over_charge() -> None: menu = ( Action(kind="charge", target_id="orc"), Action(kind="attack", target_id="orc"), ) assert fallback_default(menu) == (Action(kind="attack", target_id="orc"),) def test_fallback_default_prefers_charge_over_move() -> None: menu = ( Action(kind="move", target_id="orc"), Action(kind="charge", target_id="orc"), ) assert fallback_default(menu) == (Action(kind="charge", target_id="orc"),) def test_fallback_default_falls_back_to_first() -> None: menu = (Action(kind="total_defense"), Action(kind="delay")) assert fallback_default(menu) == (Action(kind="total_defense"),) def test_fallback_default_empty_menu_returns_wait() -> None: assert fallback_default(()) == (Action(kind="wait"),) # ---- NLPolicy integration tests --------------------------------------------- def test_nl_policy_parse_ok(builder: PromptBuilder) -> None: """Parse OK path with valid JSON from FakeLLMClient.""" script = ['[{"kind": "move", "destination": [2, 3]}]'] client = _make_client(script, builder=builder) policy = _make_policy(client, builder) actions = policy(_obs()) assert actions == (Action(kind="move", destination=(2, 3)),) assert client.stats.calls == 1 def test_nl_policy_retry_then_succeeds() -> None: """One bad reply, then a good one → parsed successfully.""" builder = _builder() script = ["not json", '[{"kind": "move", "destination": [2, 3]}]'] client = _make_client(script, builder=builder) policy = _make_policy(client, builder) actions = policy(_obs()) assert actions == (Action(kind="move", destination=(2, 3)),) assert client.stats.calls == 2 def test_nl_policy_retry_then_fallback() -> None: """Two bad replies → fallback used.""" builder = _builder() script = ["not json", "not json"] client = _make_client(script, builder=builder) policy = _make_policy(client, builder) # Minimal obs menu is (total_defense, delay) actions = policy(_obs()) # fallback_default picks first decisive action, but menu has only defensive assert actions == (Action(kind="total_defense"),) assert client.stats.calls == 2 def test_nl_policy_prompt_version_mismatch() -> None: """Client default prompt_version vs builder pack version → ValueError.""" builder = _builder() # Use default prompt_version ("fixture:fake:0") which differs from pack client = FakeLLMClient(script=['[{"kind": "wait"}]']) with pytest.raises(ValueError, match=r"prompt_version"): _make_policy(client, builder) def test_nl_policy_script_exhausted_propagates() -> None: """ScriptExhaustedError from FakeLLMClient propagates (not caught).""" builder = _builder() client = _make_client([], builder=builder) policy = _make_policy(client, builder) with pytest.raises(ScriptExhaustedError): policy(_obs()) def test_nl_policy_shared_cache_hit() -> None: """Two clients sharing a ResponseCache: second call hits cache.""" builder = _builder() cache = ResponseCache() # First client has one scripted response client1 = _make_client(['[{"kind": "wait"}]'], builder=builder, cache=cache) # Second client has NO script; relies on cache hit client2 = _make_client([], builder=builder, cache=cache) policy1 = _make_policy(client1, builder) policy2 = _make_policy(client2, builder) obs = _obs() actions1 = policy1(obs) actions2 = policy2(obs) assert actions2 == actions1 == (Action(kind="wait"),) # Second call was a cache hit (no ScriptExhaustedError) assert client2.stats.cache_hits == 1 assert client1.stats.cache_misses == 1 # ---- P5.2 repair loop -------------------------------------------------------- def test_nl_policy_stats_defaults_to_zero() -> None: """A fresh NLPolicy reports zero calls until the first decision.""" builder = _builder() client = _make_client(['[{"kind": "wait"}]'], builder=builder) policy = _make_policy(client, builder) assert policy.stats == NLPolicyStats() assert policy.stats.calls == 0 def test_nl_policy_max_repairs_zero_falls_back_immediately() -> None: """max_repairs=0 disables repair: first bad reply triggers fallback.""" builder = _builder() client = _make_client(["bad one"], builder=builder) policy = _make_policy(client, builder, max_repairs=0) actions = policy(_obs()) assert actions == (Action(kind="total_defense"),) assert policy.stats.calls == 1 assert policy.stats.repairs_attempted == 0 assert policy.stats.repairs_succeeded == 0 assert policy.stats.fallbacks == 1 def test_nl_policy_max_repairs_two_repairs_then_succeeds() -> None: """Two invalid replies are repaired, then the valid reply is parsed.""" builder = _builder() script = ["bad one", "bad two", '[{"kind": "move", "destination": [2, 3]}]'] client = _make_client(script, builder=builder) policy = _make_policy(client, builder, max_repairs=2) actions = policy(_obs()) assert actions == (Action(kind="move", destination=(2, 3)),) assert policy.stats.calls == 3 assert policy.stats.repairs_attempted == 2 assert policy.stats.repairs_succeeded == 1 assert policy.stats.fallbacks == 0 def test_nl_policy_max_repairs_two_then_falls_back() -> None: """Past max_repairs the policy stops repairing and falls back.""" builder = _builder() script = ["bad one", "bad two", "bad three"] client = _make_client(script, builder=builder) policy = _make_policy(client, builder, max_repairs=2) actions = policy(_obs()) assert actions == (Action(kind="total_defense"),) assert policy.stats.calls == 3 assert policy.stats.repairs_attempted == 2 assert policy.stats.repairs_succeeded == 0 assert policy.stats.fallbacks == 1 def test_nl_policy_repair_trail_accumulates() -> None: """Each retry prompt carries the full accumulated error trail.""" builder = _builder() script = ["bad one", "bad two", '[{"kind": "wait"}]'] client = _RecordingClient( script=script, prompt_version=builder.prompt_version("nl_default"), ) policy = _make_policy(client, builder, max_repairs=2) actions = policy(_obs()) assert actions == (Action(kind="wait"),) # First call: bare prompt with no trail. assert len(client.prompts) == 3 assert "-- previous reply invalid" not in client.prompts[0] # First retry: trail carries attempt 1 only. assert "attempt 1" in client.prompts[1] assert "attempt 2" not in client.prompts[1] assert "Return a valid JSON action sequence." in client.prompts[1] # Second retry: trail carries attempts 1 AND 2, in order. attempt1 = client.prompts[2].index("-- previous reply invalid (attempt 1) --") attempt2 = client.prompts[2].index("-- previous reply invalid (attempt 2) --") assert attempt1 < attempt2 # The base prompt is preserved at the head of every retry. assert client.prompts[1].startswith(client.prompts[0]) def test_nl_policy_max_repairs_negative_rejected() -> None: """A negative max_repairs is a configuration error, not a fallback.""" builder = _builder() client = _make_client(['[{"kind": "wait"}]'], builder=builder) with pytest.raises(ValueError, match=r"max_repairs must be >= 0, got -1"): _make_policy(client, builder, max_repairs=-1) def test_nl_policy_stats_after_direct_success() -> None: """A clean first-try success leaves the repair counters untouched.""" builder = _builder() client = _make_client(['[{"kind": "wait"}]'], builder=builder) policy = _make_policy(client, builder) actions = policy(_obs()) assert actions == (Action(kind="wait"),) assert policy.stats.calls == 1 assert policy.stats.repairs_attempted == 0 assert policy.stats.repairs_succeeded == 0 assert policy.stats.fallbacks == 0