feat(phase2): provider-agnostic LLM client + composite cache (P1b)

2026-09-22 09:45:18 +02:00
parent f0617a0de6
commit de4645838b
3 changed files with 916 additions and 0 deletions
@@ -0,0 +1,378 @@
+"""Phase-2 LLM client layer (docs/phase-2 P1.1/P1.2, decisions D6/D10).
+
+``client.py`` wraps every LLM call with the shared safeguards: composite
+cache (D6: model, params, prompt_version, schema_version, prompt hash),
+cost/token budgets + circuit breaker (P1.2, section 13), retry/backoff on
+transient provider failures, and ``FakeLLMClient`` (P1.1) to drive battles
+fully offline. These tests pin the ledger, the budgets and the breaker
+behaviour without any network.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from pf1e_simulator.client import (
+    BudgetExceededError,
+    CircuitBreaker,
+    CircuitOpenError,
+    ClientParams,
+    ClientStats,
+    Completion,
+    CostBudget,
+    FakeLLMClient,
+    LLMClient,
+    Pricing,
+    ResponseCache,
+    RetryExhaustedError,
+    ScriptExhaustedError,
+    TokenUsage,
+    estimate_tokens,
+    response_cache_key,
+)
+
+_PARAMS = ClientParams()
+_PROMPT = "observation rendered prompt v1"
+
+
+class _UnusedClient(LLMClient):
+    """Concrete client whose provider is never invoked (validation tests)."""
+
+    def _call_once(self, prompt: str, *, params: ClientParams) -> Completion:
+        del prompt, params  # validation-only: the provider never runs
+        msg = "provider must never be called"
+        raise AssertionError(msg)
+
+
+# ── composite cache key (D6) ───────────────────────────────────────────────
+
+
+def test_cache_key_is_composite_and_deterministic() -> None:
+    """Each D6 component changes the key; identical inputs are stable."""
+    base = response_cache_key(
+        model="m",
+        params=_PARAMS,
+        prompt_version="pack:1.0.0:nl:abc",
+        schema_version=1,
+        prompt=_PROMPT,
+    )
+    assert (
+        response_cache_key(
+            model="m",
+            params=_PARAMS,
+            prompt_version="pack:1.0.0:nl:abc",
+            schema_version=1,
+            prompt=_PROMPT,
+        )
+        == base
+    )
+    assert (
+        response_cache_key(
+            model="other",
+            params=_PARAMS,
+            prompt_version="pack:1.0.0:nl:abc",
+            schema_version=1,
+            prompt=_PROMPT,
+        )
+        != base
+    )
+    assert (
+        response_cache_key(
+            model="m",
+            params=ClientParams(temperature=0.7),
+            prompt_version="pack:1.0.0:nl:abc",
+            schema_version=1,
+            prompt=_PROMPT,
+        )
+        != base
+    )
+    assert (
+        response_cache_key(
+            model="m",
+            params=_PARAMS,
+            prompt_version="pack:9.9.9:nl:xyz",
+            schema_version=1,
+            prompt=_PROMPT,
+        )
+        != base
+    )
+    assert (
+        response_cache_key(
+            model="m",
+            params=_PARAMS,
+            prompt_version="pack:1.0.0:nl:abc",
+            schema_version=2,
+            prompt=_PROMPT,
+        )
+        != base
+    )
+    assert (
+        response_cache_key(
+            model="m",
+            params=_PARAMS,
+            prompt_version="pack:1.0.0:nl:abc",
+            schema_version=1,
+            prompt="a different observation",
+        )
+        != base
+    )
+
+
+def test_response_cache_round_trip_and_counters() -> None:
+    """get/put round-trip; hits and misses are counted separately."""
+    cache = ResponseCache(shard_count=4)
+    key = response_cache_key(
+        model="m",
+        params=_PARAMS,
+        prompt_version="pack:1.0.0:nl:abc",
+        schema_version=1,
+        prompt=_PROMPT,
+    )
+    assert cache.get(key) is None
+    assert cache.hits == 0
+    assert cache.misses == 1
+    cache.put(key, "action json 1")
+    assert cache.get(key) == "action json 1"
+    assert cache.hits == 1
+    assert cache.misses == 1
+
+
+def test_response_cache_clear_drops_entries_and_counters() -> None:
+    """clear() empties every shard and resets the counters (tests only)."""
+    cache = ResponseCache(shard_count=2)
+    key = response_cache_key(
+        model="m",
+        params=_PARAMS,
+        prompt_version="p:1:nl:a",
+        schema_version=1,
+        prompt=_PROMPT,
+    )
+    cache.put(key, "action json 1")
+    assert cache.get(key) == "action json 1"
+    cache.clear()
+    assert cache.get(key) is None
+    assert cache.hits == 0
+    assert cache.misses == 1
+
+
+def test_response_cache_requires_positive_shard_count() -> None:
+    """A zero shard count is a data error, not a silent default."""
+    with pytest.raises(ValueError, match="shard_count"):
+        ResponseCache(shard_count=0)
+
+
+# ── FakeLLMClient: scripted offline behaviour (P1.1) ───────────────────────
+
+
+def test_fake_serves_script_in_order() -> None:
+    """The fake returns scripted responses one per successful call."""
+    client = FakeLLMClient(script=("action a", "action b"))
+    assert client.complete("observation one") == "action a"
+    assert client.complete("observation two") == "action b"
+    assert client.stats.calls == 2
+
+
+def test_fake_script_exhaustion_raises_typed_error() -> None:
+    """A call past the script is a test-authoring bug, loudly reported."""
+    client = FakeLLMClient(script=("action a",))
+    assert client.complete("observation one") == "action a"
+    with pytest.raises(ScriptExhaustedError, match="script exhausted"):
+        client.complete("observation two")
+
+
+def test_fake_reports_estimated_token_usage() -> None:
+    """Token usage is estimated from character counts (no provider metric)."""
+    client = FakeLLMClient(script=("0123456789",))
+    client.complete(_PROMPT)
+    stats = client.stats
+    assert stats.prompt_tokens == estimate_tokens(_PROMPT)
+    assert stats.completion_tokens == estimate_tokens("0123456789")
+    assert isinstance(stats, ClientStats)
+
+
+# ── cache integration on the client (D6) ───────────────────────────────────
+
+
+def test_client_cache_hit_consumes_no_script() -> None:
+    """A cached response is returned without a provider call."""
+    client = FakeLLMClient(script=("action a", "action b"))
+    first = client.complete(_PROMPT)
+    second = client.complete(_PROMPT)
+    assert first == second == "action a"
+    assert client.calls == 1  # the second call came from the cache
+    assert client.cache_misses == 1
+    assert client.cache_hits == 1
+
+
+def test_client_cache_miss_on_different_prompt() -> None:
+    """A different prompt is a different cache key: script advances."""
+    client = FakeLLMClient(script=("action a", "action b"))
+    assert client.complete(_PROMPT) == "action a"
+    assert client.complete("another observation") == "action b"
+    assert client.calls == 2
+
+
+def test_client_cache_respects_prompt_version() -> None:
+    """The composite key includes prompt_version: packs do not collide."""
+    cache = ResponseCache()
+    a = FakeLLMClient(
+        script=("action a",),
+        prompt_version="pack:1.0.0:nl:aaa",
+        cache=cache,
+    )
+    b = FakeLLMClient(
+        script=("action b",),
+        prompt_version="pack:1.0.0:nl:bbb",
+        cache=cache,
+    )
+    assert a.complete(_PROMPT) == "action a"
+    assert b.complete(_PROMPT) == "action b"
+    assert a.calls == 1
+    assert b.calls == 1
+
+
+def test_client_cache_respects_model() -> None:
+    """The composite key includes the model name."""
+    cache = ResponseCache()
+    a = FakeLLMClient(script=("action a",), model="model-a", cache=cache)
+    b = FakeLLMClient(script=("action b",), model="model-b", cache=cache)
+    assert a.complete(_PROMPT) == "action a"
+    assert b.complete(_PROMPT) == "action b"
+    assert a.calls == 1
+    assert b.calls == 1
+
+
+# ── retry/backoff (P1.2) ───────────────────────────────────────────────────
+
+
+def test_retry_recovers_after_transient_failures() -> None:
+    """ProviderError is retried; the script is only consumed by successes."""
+    client = FakeLLMClient(
+        script=("action a",),
+        failures=2,
+        retries=3,
+    )
+    assert client.complete(_PROMPT) == "action a"
+    assert client.calls == 3  # two failures + one success
+
+
+def test_retry_exhausted_after_too_many_failures() -> None:
+    """Past the retry budget, a typed RetryExhaustedError is raised."""
+    client = FakeLLMClient(script=("action a",), failures=5, retries=2)
+    with pytest.raises(RetryExhaustedError, match="after 3 attempt"):
+        client.complete(_PROMPT)
+
+
+def test_retry_counter_requires_positive_value() -> None:
+    """A negative retry count is a data error."""
+    with pytest.raises(ValueError, match="retries"):
+        FakeLLMClient(script=("a",), retries=-1)
+
+
+# ── circuit breaker (P1.2, section 13) ─────────────────────────────────────
+
+
+def test_breaker_opens_refuses_probes_and_recovers() -> None:
+    """Open after N failures: refusals, then a probe that closes on success."""
+    breaker = CircuitBreaker(failure_threshold=2, open_refusals=1)
+    client = FakeLLMClient(
+        script=("action a",),
+        failures=2,  # needs two attempts before real service
+        retries=0,
+        breaker=breaker,
+    )
+    with pytest.raises(RetryExhaustedError):
+        client.complete(_PROMPT)  # failure 1: not open yet
+    with pytest.raises(RetryExhaustedError):
+        client.complete(_PROMPT)  # failure 2: threshold reached, opens
+    assert breaker.is_open
+    with pytest.raises(CircuitOpenError, match="circuit breaker open"):
+        client.complete(_PROMPT)  # refused
+    assert client.complete(_PROMPT) == "action a"  # probe succeeds now
+    assert not breaker.is_open
+
+
+def test_breaker_probe_failure_reopens() -> None:
+    """A failed probe reopens the breaker for another refusal window."""
+    breaker = CircuitBreaker(failure_threshold=1, open_refusals=1)
+    client = FakeLLMClient(
+        script=("action a",),
+        failures=10,  # every attempt fails, probe included
+        retries=0,
+        breaker=breaker,
+    )
+    with pytest.raises(RetryExhaustedError):
+        client.complete(_PROMPT)
+    assert breaker.is_open
+    with pytest.raises(CircuitOpenError):
+        client.complete(_PROMPT)
+    with pytest.raises(RetryExhaustedError):
+        client.complete(_PROMPT)  # the probe fails
+    assert breaker.is_open
+    with pytest.raises(CircuitOpenError):
+        client.complete(_PROMPT)  # reopened: refuses again
+
+
+def test_breaker_validation() -> None:
+    """Breaker thresholds must be sensible numbers."""
+    with pytest.raises(ValueError, match="failure_threshold"):
+        CircuitBreaker(failure_threshold=0)
+    with pytest.raises(ValueError, match="open_refusals"):
+        CircuitBreaker(open_refusals=-1)
+
+
+# ── cost/token budgets (P1.2) ──────────────────────────────────────────────
+
+
+def test_call_budget_ceiling() -> None:
+    """max_calls refuses further calls with a typed error."""
+    client = FakeLLMClient(
+        script=("a", "b", "c"),
+        budget=CostBudget(max_calls=1),
+    )
+    assert client.complete(_PROMPT) == "a"
+    with pytest.raises(BudgetExceededError, match="call budget exceeded"):
+        client.complete("a different observation")
+
+
+def test_token_budget_ceiling() -> None:
+    """A completion over max_total_tokens is rejected and not cached."""
+    client = FakeLLMClient(
+        script=("x" * 400,),  # ~100 estimated completion tokens
+        budget=CostBudget(max_total_tokens=50),
+    )
+    with pytest.raises(BudgetExceededError, match="token budget exceeded"):
+        client.complete(_PROMPT)
+
+
+def test_cost_budget_ceiling_and_ledger() -> None:
+    """Estimated USD cost is accumulated and capped by max_cost_usd."""
+    pricing = Pricing(prompt_per_1k_usd=1.0, completion_per_1k_usd=1.0)
+    long_prompt = "p" * 2000  # ~500 prompt tokens ≈ 0.5 USD
+    client = FakeLLMClient(
+        script=("c" * 2000, "c" * 2000),  # ~500 completion tokens each ≈ 0.5 USD
+        pricing=pricing,
+        budget=CostBudget(max_cost_usd=1.0),
+    )
+    assert client.complete(long_prompt) == "c" * 2000
+    assert client.cost_usd == pytest.approx(1.0, abs=1e-6)
+    with pytest.raises(BudgetExceededError, match="cost budget exceeded"):
+        client.complete("q" * 2000)
+    assert client.cost_usd == pytest.approx(2.0, abs=1e-6)
+
+
+def test_backoff_must_be_positive() -> None:
+    """A negative backoff is a data error (validation-only client)."""
+    with pytest.raises(ValueError, match="backoff_base_s"):
+        _UnusedClient(
+            model="m",
+            prompt_version="p:1:nl:a",
+            backoff_base_s=-1.0,
+        )
+
+
+def test_token_usage_total() -> None:
+    """TokenUsage.total is the sum of both counters."""
+    usage = TokenUsage(prompt_tokens=3, completion_tokens=4)
+    assert usage.total == 7
+    assert estimate_tokens("abcd") >= 1