kyeongmin
896f273ffa
Replaces #84 UI-noise removal plan with positive operational-alert contract. Five-axis stack lands together: (1) default model literal moved to current Opus-family ID, (2) Anthropic SDK error classifier mapping exceptions to quota/billing/auth/other, (3) api_error_kind plumbed through ai_repair_status summary + per-record retention, (4) Step 0 preflight ping gated under ai_fallback_enabled (default OFF preserved) with fail-fast on invalid model/key, (5) frontend formatter rewritten to surface only operational quota/billing/auth toasts (non-operational paths return null per feedback_auto_pipeline_first silent-pipeline policy). u1 - default model literal claude-opus-4-6-20250415 -> claude-opus-4-7 (src/config.py + tests/test_phase_z2_ai_fallback_config.py lock mirror) u2 - classify_operational_error type+status_code dispatch + Step 12 api_error_kind stamp on except path (src/phase_z2_ai_fallback/client.py + src/phase_z2_ai_fallback/step12.py + tests/phase_z2_ai_fallback/test_step12.py) u3 - _summarize_ai_repair_status aggregates api_error_kinds {quota,billing, auth,other}; error_records[i].api_error_kind retained per-record (src/phase_z2_pipeline.py + tests/test_imp47b_failure_surface.py) u4 - _run_step0_ai_preflight + Step0PreflightError; preflight only fires when ai_fallback_enabled=true; one-token ping; invalid key/model => setup failure before Step 1 (src/phase_z2_pipeline.py + tests/phase_z2/test_pipeline_step0_preflight.py NEW) u5 - AiRepairStatus.api_error_kinds? interface + formatAiRepairHumanReview Message rewritten: operational quota/billing/auth -> Korean copy verbatim from issue body (tie-break quota -> billing -> auth); validation/coverage_violated/unsupported_kind/generic-other/legacy payload -> null (Front/client/src/services/designAgentApi.ts + Front/client/tests/imp47b_human_review_toast.test.tsx) Guardrails respected: - feedback_demo_env_toggle_policy: default OFF preserved; preflight skipped when ai_fallback_enabled=false (test_preflight_skipped_when_disabled asserts anthropic.Anthropic() not called). - feedback_auto_pipeline_first: non-operational AI failures stay silent; only quota/billing/auth reach user toast. - feedback_ai_isolation_contract: AI remains fallback-only; no normal-path migration; MDX preserved. - project_imp46_carveout_caveat: cache_key/fingerprints fields untouched on every record; no overlap with #62 cache region. - feedback_no_hardcoding: zero MDX-sample-specific literals; classifier dispatch by SDK type, not by string parsing. - feedback_artifact_status_naming: operational toast scoped to alert axis, not overall PASS signal. Tests: - Targeted u1+u2+u3+u4: 63 passed - u5 vitest (Front/): 10/10 passed - tests/phase_z2_ai_fallback dir regression: 240 passed - tests/phase_z2 dir regression: 323 passed - IMP-92-adjacent (-k "imp47b or ai_fallback or preflight or step12 or step0"): 299 passed (808 deselected) - u1 baseline lock (test_client_mock.py): 8 passed Zero failures, zero regressions outside scope. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
215 lines
8.3 KiB
Python
215 lines
8.3 KiB
Python
"""IMP-92 u4 — Step 0 AI preflight unit tests.
|
|
|
|
Scope (Stage 2 plan, u4):
|
|
- ``settings.ai_fallback_enabled=False`` → preflight short-circuits to
|
|
``"skipped"`` without instantiating ``anthropic.Anthropic`` (PZ-1
|
|
AI=0 normal path + ``feedback_demo_env_toggle_policy`` default-OFF).
|
|
- ``settings.ai_fallback_enabled=True`` + valid (key, model) → preflight
|
|
returns ``"passed"`` after a 1-token ``messages.create`` ping.
|
|
- Persistent setup errors (Authentication / PermissionDenied /
|
|
NotFound) raise ``Step0PreflightError`` so boot fails fast.
|
|
- Transient errors (RateLimit / InternalServer) are recorded as
|
|
``"transient"`` without failing boot.
|
|
|
|
Cross-references:
|
|
- u1 default model literal: ``src/config.py:20``
|
|
+ ``tests/test_phase_z2_ai_fallback_config.py:5,31``
|
|
- u2 SDK operational classifier:
|
|
``src/phase_z2_ai_fallback/client.py:46``
|
|
+ ``tests/phase_z2_ai_fallback/test_step12.py``
|
|
- u3 ``api_error_kind`` summary plumbing:
|
|
``src/phase_z2_pipeline.py:_summarize_ai_repair_status``
|
|
+ ``tests/test_imp47b_failure_surface.py``
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
import anthropic
|
|
import httpx
|
|
import pytest
|
|
|
|
from src import phase_z2_pipeline as pipeline_mod
|
|
from src.config import settings
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_settings():
|
|
snapshot = settings.model_dump()
|
|
yield
|
|
for key, value in snapshot.items():
|
|
setattr(settings, key, value)
|
|
|
|
|
|
def _ok_response() -> SimpleNamespace:
|
|
return SimpleNamespace(content=[SimpleNamespace(text="")])
|
|
|
|
|
|
def _status_error(
|
|
cls: type[anthropic.APIStatusError],
|
|
status_code: int,
|
|
message: str,
|
|
) -> anthropic.APIStatusError:
|
|
req = httpx.Request("POST", "https://api.anthropic.com/v1/messages")
|
|
return cls(
|
|
message=message,
|
|
response=httpx.Response(status_code, request=req),
|
|
body=None,
|
|
)
|
|
|
|
|
|
def test_preflight_skipped_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", False)
|
|
spy = MagicMock()
|
|
monkeypatch.setattr(anthropic, "Anthropic", spy)
|
|
result = pipeline_mod._run_step0_ai_preflight()
|
|
assert result["status"] == "skipped"
|
|
assert result["reason"] == "ai_fallback_disabled"
|
|
assert result["model"] == settings.ai_fallback_model
|
|
spy.assert_not_called()
|
|
|
|
|
|
def test_preflight_passed_when_enabled_with_valid_credentials(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.return_value = _ok_response()
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
result = pipeline_mod._run_step0_ai_preflight()
|
|
assert result == {
|
|
"status": "passed",
|
|
"model": settings.ai_fallback_model,
|
|
}
|
|
fake_client.messages.create.assert_called_once()
|
|
kwargs = fake_client.messages.create.call_args.kwargs
|
|
assert kwargs["model"] == settings.ai_fallback_model
|
|
assert kwargs["max_tokens"] == 1
|
|
|
|
|
|
def test_preflight_fail_fast_on_invalid_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.AuthenticationError, 401, "invalid x-api-key"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
with pytest.raises(pipeline_mod.Step0PreflightError) as ei:
|
|
pipeline_mod._run_step0_ai_preflight()
|
|
assert "AuthenticationError" in str(ei.value)
|
|
|
|
|
|
def test_preflight_fail_fast_on_invalid_model(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.NotFoundError, 404, "model not found"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
with pytest.raises(pipeline_mod.Step0PreflightError) as ei:
|
|
pipeline_mod._run_step0_ai_preflight()
|
|
msg = str(ei.value)
|
|
assert "NotFoundError" in msg
|
|
assert settings.ai_fallback_model in msg
|
|
|
|
|
|
def test_preflight_fail_fast_on_billing_permission_denied(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.PermissionDeniedError, 403, "billing required"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
with pytest.raises(pipeline_mod.Step0PreflightError) as ei:
|
|
pipeline_mod._run_step0_ai_preflight()
|
|
assert "PermissionDeniedError" in str(ei.value)
|
|
|
|
|
|
def test_preflight_transient_rate_limit_does_not_fail_boot(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.RateLimitError, 429, "rate limited"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
result = pipeline_mod._run_step0_ai_preflight()
|
|
assert result["status"] == "transient"
|
|
assert result["model"] == settings.ai_fallback_model
|
|
assert "RateLimitError" in result["transient_error"]
|
|
|
|
|
|
def test_preflight_transient_internal_server_error_does_not_fail_boot(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.InternalServerError, 500, "upstream 500"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
result = pipeline_mod._run_step0_ai_preflight()
|
|
assert result["status"] == "transient"
|
|
assert "InternalServerError" in result["transient_error"]
|
|
|
|
|
|
def test_preflight_fail_fast_on_generic_billing_402(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""IMP-92 u4 — HTTP 402 (Payment Required) surfaces as the generic
|
|
``anthropic.APIStatusError`` (no typed subclass). The preflight MUST
|
|
dispatch by status code and raise ``Step0PreflightError`` so a
|
|
billing setup problem fails boot fast, matching the issue body's
|
|
operational contract.
|
|
"""
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.APIStatusError, 402, "payment required"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
with pytest.raises(pipeline_mod.Step0PreflightError) as ei:
|
|
pipeline_mod._run_step0_ai_preflight()
|
|
msg = str(ei.value)
|
|
assert "402" in msg
|
|
assert settings.ai_fallback_model in msg
|
|
assert "Check ANTHROPIC_API_KEY / ai_fallback_model in .env." in msg
|
|
|
|
|
|
def test_preflight_generic_status_429_treated_as_transient(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""IMP-92 u4 — a generic ``APIStatusError`` with HTTP 429 must follow
|
|
the same transient policy as the typed ``RateLimitError`` branch.
|
|
"""
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.APIStatusError, 429, "rate limited (generic)"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
result = pipeline_mod._run_step0_ai_preflight()
|
|
assert result["status"] == "transient"
|
|
assert "APIStatusError" in result["transient_error"]
|
|
|
|
|
|
def test_preflight_generic_status_5xx_treated_as_transient(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""IMP-92 u4 — a generic ``APIStatusError`` with HTTP 5xx must follow
|
|
the same transient policy as the typed ``InternalServerError`` branch.
|
|
"""
|
|
monkeypatch.setattr(settings, "ai_fallback_enabled", True)
|
|
fake_client = MagicMock()
|
|
fake_client.messages.create.side_effect = _status_error(
|
|
anthropic.APIStatusError, 503, "upstream 503 (generic)"
|
|
)
|
|
monkeypatch.setattr(anthropic, "Anthropic", lambda **kwargs: fake_client)
|
|
result = pipeline_mod._run_step0_ai_preflight()
|
|
assert result["status"] == "transient"
|
|
assert "APIStatusError" in result["transient_error"]
|