feat(#92): IMP-92 u1~u5 AI fallback config validation (model ping + operational error classification)
Replaces #84 UI-noise removal plan with positive operational-alert contract. Five-axis stack lands together: (1) default model literal moved to current Opus-family ID, (2) Anthropic SDK error classifier mapping exceptions to quota/billing/auth/other, (3) api_error_kind plumbed through ai_repair_status summary + per-record retention, (4) Step 0 preflight ping gated under ai_fallback_enabled (default OFF preserved) with fail-fast on invalid model/key, (5) frontend formatter rewritten to surface only operational quota/billing/auth toasts (non-operational paths return null per feedback_auto_pipeline_first silent-pipeline policy). u1 - default model literal claude-opus-4-6-20250415 -> claude-opus-4-7 (src/config.py + tests/test_phase_z2_ai_fallback_config.py lock mirror) u2 - classify_operational_error type+status_code dispatch + Step 12 api_error_kind stamp on except path (src/phase_z2_ai_fallback/client.py + src/phase_z2_ai_fallback/step12.py + tests/phase_z2_ai_fallback/test_step12.py) u3 - _summarize_ai_repair_status aggregates api_error_kinds {quota,billing, auth,other}; error_records[i].api_error_kind retained per-record (src/phase_z2_pipeline.py + tests/test_imp47b_failure_surface.py) u4 - _run_step0_ai_preflight + Step0PreflightError; preflight only fires when ai_fallback_enabled=true; one-token ping; invalid key/model => setup failure before Step 1 (src/phase_z2_pipeline.py + tests/phase_z2/test_pipeline_step0_preflight.py NEW) u5 - AiRepairStatus.api_error_kinds? interface + formatAiRepairHumanReview Message rewritten: operational quota/billing/auth -> Korean copy verbatim from issue body (tie-break quota -> billing -> auth); validation/coverage_violated/unsupported_kind/generic-other/legacy payload -> null (Front/client/src/services/designAgentApi.ts + Front/client/tests/imp47b_human_review_toast.test.tsx) Guardrails respected: - feedback_demo_env_toggle_policy: default OFF preserved; preflight skipped when ai_fallback_enabled=false (test_preflight_skipped_when_disabled asserts anthropic.Anthropic() not called). - feedback_auto_pipeline_first: non-operational AI failures stay silent; only quota/billing/auth reach user toast. - feedback_ai_isolation_contract: AI remains fallback-only; no normal-path migration; MDX preserved. - project_imp46_carveout_caveat: cache_key/fingerprints fields untouched on every record; no overlap with #62 cache region. - feedback_no_hardcoding: zero MDX-sample-specific literals; classifier dispatch by SDK type, not by string parsing. - feedback_artifact_status_naming: operational toast scoped to alert axis, not overall PASS signal. Tests: - Targeted u1+u2+u3+u4: 63 passed - u5 vitest (Front/): 10/10 passed - tests/phase_z2_ai_fallback dir regression: 240 passed - tests/phase_z2 dir regression: 323 passed - IMP-92-adjacent (-k "imp47b or ai_fallback or preflight or step12 or step0"): 299 passed (808 deselected) - u1 baseline lock (test_client_mock.py): 8 passed Zero failures, zero regressions outside scope. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,20 +1,28 @@
|
||||
// IMP-47B u11 — Frontend ai_repair_status notification surfacing.
|
||||
// IMP-92 u5 — Frontend AI repair operational-only formatter test surface.
|
||||
//
|
||||
// Scope (Stage 2 unit u11 contract):
|
||||
// 1) loadRun → RunMeta.ai_repair_status exposes the u8 step20 payload.
|
||||
// 2) formatAiRepairHumanReviewMessage(...) returns user-facing notification
|
||||
// text on the three failure axes (error / coverage_violated /
|
||||
// unsupported_kind) and returns null on success / no-AI paths.
|
||||
// Scope (Stage 2 unit u5 contract):
|
||||
// 1) formatAiRepairHumanReviewMessage(...) surfaces a user-facing toast
|
||||
// ONLY on the three operational Anthropic API error kinds (quota /
|
||||
// billing / auth) classified by Step 12 u2
|
||||
// (classify_operational_error) and aggregated through u3
|
||||
// ai_repair_status.api_error_kinds.
|
||||
// 2) Non-operational AI failures (validation / coverage_violated /
|
||||
// unsupported_kind / generic "other") return null so the
|
||||
// auto-pipeline stays silent per feedback_auto_pipeline_first and
|
||||
// the #84 operational-vs-non-operational replacement-plan contract.
|
||||
// 3) Replaces the prior IMP-47B u11 surface — previously rendered toasts
|
||||
// for error / coverage_violated / unsupported_kind. After IMP-92 the
|
||||
// ONLY operational reaches the user; non-operational stays silent.
|
||||
//
|
||||
// Pure-function unit test (no React Testing Library required — vitest is
|
||||
// already in devDependencies; @testing-library/* is NOT installed). The
|
||||
// Home.tsx wiring is a 2-line site that calls this helper after
|
||||
// setRunMeta(...); covering the helper covers the user-visible message text
|
||||
// directly without DOM rendering.
|
||||
// Home.tsx wiring is a 2-line site (`Home.tsx:438`) that calls this helper
|
||||
// after `setRunMeta(...)`; covering the helper covers the user-visible
|
||||
// message text directly without DOM rendering.
|
||||
//
|
||||
// File extension is `.tsx` per Stage 2 unit contract path; no JSX is required
|
||||
// for these assertions but the extension allows future RTL-based tests to
|
||||
// land here without renaming.
|
||||
// The test file path is preserved from IMP-47B u11 (Stage 2 plan
|
||||
// `Front/client/tests/imp47b_human_review_toast.test.tsx`); the assertions
|
||||
// inside reflect the IMP-92 u5 operational-only contract.
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
@@ -23,7 +31,7 @@ import {
|
||||
} from "../src/services/designAgentApi";
|
||||
|
||||
const baseCounts = {
|
||||
total: 1,
|
||||
total: 0,
|
||||
applied: 0,
|
||||
no_proposal: 0,
|
||||
no_zone_match: 0,
|
||||
@@ -31,16 +39,19 @@ const baseCounts = {
|
||||
error: 0,
|
||||
};
|
||||
|
||||
describe("formatAiRepairHumanReviewMessage (IMP-47B u11)", () => {
|
||||
it("returns null when ai_repair_status is null (legacy / pre-Step12 abort)", () => {
|
||||
const zeroKinds = { quota: 0, billing: 0, auth: 0, other: 0 };
|
||||
|
||||
describe("formatAiRepairHumanReviewMessage (IMP-92 u5 — operational-only)", () => {
|
||||
it("returns null when ai_repair_status is null / undefined", () => {
|
||||
expect(formatAiRepairHumanReviewMessage(null)).toBeNull();
|
||||
expect(formatAiRepairHumanReviewMessage(undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when human_review_required=false (success / no-AI path)", () => {
|
||||
it("returns null on success / no-AI path (no operational kind present)", () => {
|
||||
const ok: AiRepairStatus = {
|
||||
status: "ok",
|
||||
counts: { ...baseCounts, total: 0 },
|
||||
counts: { ...baseCounts },
|
||||
api_error_kinds: { ...zeroKinds },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [],
|
||||
coverage_status: "ok",
|
||||
@@ -57,47 +68,127 @@ describe("formatAiRepairHumanReviewMessage (IMP-47B u11)", () => {
|
||||
expect(formatAiRepairHumanReviewMessage(applied)).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces AI call failures with count + frame/manual guidance", () => {
|
||||
const errored: AiRepairStatus = {
|
||||
it("surfaces quota operational alert (Anthropic 429 / RateLimitError)", () => {
|
||||
const ai: AiRepairStatus = {
|
||||
status: "error",
|
||||
counts: { ...baseCounts, total: 2, error: 2 },
|
||||
api_error_kinds: { quota: 2, billing: 0, auth: 0, other: 0 },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [
|
||||
{ unit_index: 0, source_section_ids: ["03-1"], error: "timeout" },
|
||||
{ unit_index: 1, source_section_ids: ["03-2"], error: "validation" },
|
||||
{
|
||||
unit_index: 0,
|
||||
source_section_ids: ["03-1"],
|
||||
error: "RateLimitError: rate_limit_exceeded",
|
||||
api_error_kind: "quota",
|
||||
},
|
||||
{
|
||||
unit_index: 1,
|
||||
source_section_ids: ["03-2"],
|
||||
error: "RateLimitError: rate_limit_exceeded",
|
||||
api_error_kind: "quota",
|
||||
},
|
||||
],
|
||||
coverage_status: "ok",
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(errored);
|
||||
const msg = formatAiRepairHumanReviewMessage(ai);
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg).toContain("AI 재구성 호출 실패");
|
||||
expect(msg).toContain("API quota");
|
||||
expect(msg).toContain("충전 필요");
|
||||
expect(msg).toContain("2");
|
||||
expect(msg).toContain("다른 frame 선택 또는 수동 편집 필요");
|
||||
});
|
||||
|
||||
it("surfaces coverage violations with the dropped section ids", () => {
|
||||
const dropped: AiRepairStatus = {
|
||||
it("surfaces billing operational alert (Anthropic 402 / PermissionDeniedError)", () => {
|
||||
const ai: AiRepairStatus = {
|
||||
status: "error",
|
||||
counts: { ...baseCounts, total: 1, error: 1 },
|
||||
api_error_kinds: { quota: 0, billing: 1, auth: 0, other: 0 },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [
|
||||
{
|
||||
unit_index: 0,
|
||||
source_section_ids: ["03-1"],
|
||||
error: "PermissionDeniedError: insufficient credits",
|
||||
api_error_kind: "billing",
|
||||
},
|
||||
],
|
||||
coverage_status: "ok",
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(ai);
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg).toContain("API billing");
|
||||
expect(msg).toContain("결제 정보 확인");
|
||||
expect(msg).toContain("1");
|
||||
});
|
||||
|
||||
it("surfaces auth operational alert (Anthropic 401 / AuthenticationError)", () => {
|
||||
const ai: AiRepairStatus = {
|
||||
status: "error",
|
||||
counts: { ...baseCounts, total: 1, error: 1 },
|
||||
api_error_kinds: { quota: 0, billing: 0, auth: 1, other: 0 },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [
|
||||
{
|
||||
unit_index: 0,
|
||||
source_section_ids: ["03-1"],
|
||||
error: "AuthenticationError: invalid x-api-key",
|
||||
api_error_kind: "auth",
|
||||
},
|
||||
],
|
||||
coverage_status: "ok",
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(ai);
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg).toContain("API key 무효");
|
||||
expect(msg).toContain(".env");
|
||||
expect(msg).toContain("1");
|
||||
});
|
||||
|
||||
it("returns null on generic non-operational 'other' API error (silent)", () => {
|
||||
const ai: AiRepairStatus = {
|
||||
status: "error",
|
||||
counts: { ...baseCounts, total: 1, error: 1 },
|
||||
api_error_kinds: { quota: 0, billing: 0, auth: 0, other: 1 },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [
|
||||
{
|
||||
unit_index: 0,
|
||||
source_section_ids: ["03-1"],
|
||||
error: "ValidationError: proposal failed schema",
|
||||
api_error_kind: "other",
|
||||
},
|
||||
],
|
||||
coverage_status: "ok",
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
expect(formatAiRepairHumanReviewMessage(ai)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null on coverage_violated (non-operational, silent)", () => {
|
||||
const ai: AiRepairStatus = {
|
||||
status: "coverage_violated",
|
||||
counts: { ...baseCounts, total: 1, applied: 1 },
|
||||
api_error_kinds: { ...zeroKinds },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [],
|
||||
coverage_status: "violated",
|
||||
dropped_section_ids: ["03-2"],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(dropped);
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg).toContain("콘텐츠 누락");
|
||||
expect(msg).toContain("03-2");
|
||||
expect(msg).toContain("다른 frame 선택 또는 수동 편집 필요");
|
||||
expect(formatAiRepairHumanReviewMessage(ai)).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces unsupported proposal kinds with the unsupported count", () => {
|
||||
const unsupported: AiRepairStatus = {
|
||||
it("returns null on unsupported_kind (non-operational, silent)", () => {
|
||||
const ai: AiRepairStatus = {
|
||||
status: "unsupported_kind",
|
||||
counts: { ...baseCounts, total: 1, unsupported_kind: 1 },
|
||||
api_error_kinds: { ...zeroKinds },
|
||||
unsupported_kind_records: [
|
||||
{
|
||||
unit_index: 0,
|
||||
@@ -110,26 +201,57 @@ describe("formatAiRepairHumanReviewMessage (IMP-47B u11)", () => {
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(unsupported);
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg).toContain("AI 제안 형식 미지원");
|
||||
expect(msg).toContain("1");
|
||||
expect(msg).toContain("다른 frame 선택 또는 수동 편집 필요");
|
||||
expect(formatAiRepairHumanReviewMessage(ai)).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to a generic human_review message on unknown status enums", () => {
|
||||
const future: AiRepairStatus = {
|
||||
status: "future_axis_not_yet_mapped",
|
||||
counts: { ...baseCounts, total: 0 },
|
||||
it("returns null on legacy ai_repair_status without api_error_kinds (pre-u3 runs)", () => {
|
||||
// Backward-compat: payloads emitted before u3 plumbing landed don't
|
||||
// carry api_error_kinds. Operational-only contract treats the absence
|
||||
// as "no operational signal" → silent (no toast).
|
||||
const legacy: AiRepairStatus = {
|
||||
status: "error",
|
||||
counts: { ...baseCounts, total: 1, error: 1 },
|
||||
// api_error_kinds intentionally omitted
|
||||
unsupported_kind_records: [],
|
||||
error_records: [],
|
||||
error_records: [
|
||||
{ unit_index: 0, source_section_ids: ["03-1"], error: "timeout" },
|
||||
],
|
||||
coverage_status: "ok",
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(future);
|
||||
expect(formatAiRepairHumanReviewMessage(legacy)).toBeNull();
|
||||
});
|
||||
|
||||
it("prioritises quota when multiple operational kinds co-occur", () => {
|
||||
// Defensive: a run that accumulated quota + billing errors across
|
||||
// multiple AI repair attempts surfaces the quota line first (the
|
||||
// most-frequently actionable per the issue body ordering).
|
||||
const ai: AiRepairStatus = {
|
||||
status: "error",
|
||||
counts: { ...baseCounts, total: 2, error: 2 },
|
||||
api_error_kinds: { quota: 1, billing: 1, auth: 0, other: 0 },
|
||||
unsupported_kind_records: [],
|
||||
error_records: [
|
||||
{
|
||||
unit_index: 0,
|
||||
source_section_ids: ["03-1"],
|
||||
error: "RateLimitError",
|
||||
api_error_kind: "quota",
|
||||
},
|
||||
{
|
||||
unit_index: 1,
|
||||
source_section_ids: ["03-2"],
|
||||
error: "PermissionDeniedError",
|
||||
api_error_kind: "billing",
|
||||
},
|
||||
],
|
||||
coverage_status: "ok",
|
||||
dropped_section_ids: [],
|
||||
human_review_required: true,
|
||||
};
|
||||
const msg = formatAiRepairHumanReviewMessage(ai);
|
||||
expect(msg).not.toBeNull();
|
||||
expect(msg).toContain("human_review");
|
||||
expect(msg).toContain("future_axis_not_yet_mapped");
|
||||
expect(msg).toContain("API quota");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user