- u1: BuilderMissingError(FitError) — narrow exception aligned with pipeline catch - u2: load_frame_contracts catalog invariant + VP skip + CatalogInvariantError - u3a: audit CLI I1~I3 (partial existence / declared builder / registry membership) - u3b: audit CLI I4 (slot_payload refs vs declared/generated payload keys) - u4: lookup_v4_candidates VP filter (lookup_v4_all_judgments raw telemetry untouched) - u5: catalog invariant regression coverage + temp non-VP failure fixtures - u6: mdx04 VP routing fixture tests (sw_dependency_four_problems excluded from live) - u7: tests/conftest.py env isolation + mdx03/mdx04/mdx05 subprocess smoke Targeted 74 PASS (12.31s). Full regression 1063 PASS (87.70s). Audit CLI clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
228 lines
8.7 KiB
Python
228 lines
8.7 KiB
Python
"""Phase Z catalog invariant test — real `frame_contracts.yaml` 1:1 mapping verify.
|
|
|
|
IMP-05 L4 lock per Claude #13 §3 :
|
|
- real catalog read (purpose 자체 = real catalog 검증)
|
|
- template_id ↔ frame_id 1:1 mapping (Codex #6 terminology — 2 reference keys for same entry)
|
|
- fail fast with explicit message if catalog policy changes
|
|
|
|
Codex #5 verified : 11 templates / 11 frames, all unique = 1:1 mapping confirm (2026-05-13).
|
|
Codex #7 generalization guardrail : real catalog OK (purpose 자체) — NOT sample-hardcoding.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
PROJECT_ROOT = Path(__file__).parent.parent
|
|
CATALOG_PATH = PROJECT_ROOT / "templates" / "phase_z2" / "catalog" / "frame_contracts.yaml"
|
|
|
|
|
|
def _load_catalog() -> dict:
|
|
with CATALOG_PATH.open(encoding="utf-8") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def test_catalog_template_id_to_frame_id_one_to_one():
|
|
"""Verify each catalog entry has unique template_id + unique frame_id (1:1 reference keys).
|
|
|
|
Fails fast if the catalog policy ever drifts from this assumption — IMP-05 dedup
|
|
relies on `template_id` as the runtime key and assumes one frame per template.
|
|
"""
|
|
catalog = _load_catalog()
|
|
|
|
template_ids = []
|
|
frame_ids = []
|
|
for entry_key, entry in catalog.items():
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
tid = entry.get("template_id")
|
|
fid = entry.get("frame_id")
|
|
assert tid is not None, f"entry {entry_key} missing template_id"
|
|
assert fid is not None, f"entry {entry_key} missing frame_id"
|
|
template_ids.append(tid)
|
|
frame_ids.append(str(fid))
|
|
|
|
duplicate_templates = [t for t in template_ids if template_ids.count(t) > 1]
|
|
duplicate_frames = [f for f in frame_ids if frame_ids.count(f) > 1]
|
|
|
|
assert not duplicate_templates, (
|
|
"Phase Z catalog currently expects one template_id per frame_id; "
|
|
"update dedup policy if this changes. "
|
|
f"Duplicate template_ids found: {set(duplicate_templates)}"
|
|
)
|
|
assert not duplicate_frames, (
|
|
"Phase Z catalog currently expects one template_id per frame_id; "
|
|
"update dedup policy if this changes. "
|
|
f"Duplicate frame_ids found: {set(duplicate_frames)}"
|
|
)
|
|
assert len(template_ids) == len(frame_ids), (
|
|
"Phase Z catalog template_id count must equal frame_id count "
|
|
f"(templates={len(template_ids)}, frames={len(frame_ids)})."
|
|
)
|
|
|
|
|
|
def test_catalog_entry_count_matches_frame_count():
|
|
"""Sanity guard — each entry contributes one template_id + one frame_id."""
|
|
catalog = _load_catalog()
|
|
entry_count = sum(1 for v in catalog.values() if isinstance(v, dict))
|
|
template_count = sum(
|
|
1 for v in catalog.values()
|
|
if isinstance(v, dict) and v.get("template_id") is not None
|
|
)
|
|
frame_count = sum(
|
|
1 for v in catalog.values()
|
|
if isinstance(v, dict) and v.get("frame_id") is not None
|
|
)
|
|
assert entry_count == template_count == frame_count, (
|
|
f"catalog shape inconsistent: entries={entry_count} "
|
|
f"templates={template_count} frames={frame_count}"
|
|
)
|
|
|
|
|
|
# ──────────────────────── IMP-#85 u5 regression coverage ────────────────────────
|
|
#
|
|
# Scope (Stage 2 lock):
|
|
# - Prod catalog passes the audit CLI (run_audit) end-to-end.
|
|
# - Non-VP fixture catalogs reproduce the boot invariant (u2) + audit (u3a/u3b)
|
|
# negative paths: missing payload.builder, missing partial, undeclared
|
|
# slot_payload reference (I4 generated-key-orphan).
|
|
# - Same fixtures with `visual_pending: true` MUST be silently skipped — the
|
|
# data-driven VP scope guard from u2/u3a/u3b must not regress.
|
|
#
|
|
# Out of scope:
|
|
# - Implementing the 17 missing VP builders (별 P0 / IMP-04b backlog).
|
|
# - Visual rendering of fixture frames.
|
|
#
|
|
# Path-convention note (tests/CLAUDE.md §F-5):
|
|
# Stage 2 plan named `tests/fixtures/catalog/` but the project convention
|
|
# reserves the root `tests/fixtures/` for non-Phase-Z fixtures (creation
|
|
# requires a separate issue). Phase-Z YAML fixtures live under
|
|
# `tests/phase_z2/fixtures/`. The u5 fixtures therefore live at
|
|
# `tests/phase_z2/fixtures/catalog/`.
|
|
|
|
import yaml
|
|
|
|
from scripts.audit_frame_invariants import (
|
|
DEFAULT_CATALOG_PATH,
|
|
DEFAULT_PARTIALS_DIR,
|
|
run_audit,
|
|
)
|
|
from src import phase_z2_mapper
|
|
from src.phase_z2_mapper import (
|
|
CatalogInvariantError,
|
|
PAYLOAD_BUILDERS,
|
|
_check_catalog_builder_invariant,
|
|
)
|
|
|
|
_IMP85_FIXTURES_DIR = Path(__file__).parent / "phase_z2" / "fixtures" / "catalog"
|
|
_MISSING_BUILDER_FIXTURE = _IMP85_FIXTURES_DIR / "missing_builder_non_vp.yaml"
|
|
_UNDECLARED_SLOT_FIXTURE = _IMP85_FIXTURES_DIR / "undeclared_slot_ref_non_vp.yaml"
|
|
|
|
|
|
def _load_fixture_catalog(path: Path) -> dict:
|
|
with path.open(encoding="utf-8") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
@pytest.fixture
|
|
def _reset_catalog_cache_for_imp85():
|
|
"""Some tests below load fixture YAMLs into the boot invariant; ensure the
|
|
prod cache is untouched on entry/exit so other tests stay deterministic."""
|
|
phase_z2_mapper._CATALOG_CACHE = None
|
|
yield
|
|
phase_z2_mapper._CATALOG_CACHE = None
|
|
|
|
|
|
def test_prod_catalog_audit_clean():
|
|
"""IMP-#85 u5 — prod catalog + prod partials dir pass audit (I1-I4 clean)."""
|
|
violations = run_audit(DEFAULT_CATALOG_PATH, DEFAULT_PARTIALS_DIR)
|
|
assert violations == [], (
|
|
f"Prod catalog audit reported {len(violations)} violation(s):\n - "
|
|
+ "\n - ".join(violations)
|
|
)
|
|
|
|
|
|
def test_missing_builder_fixture_raises_catalog_invariant(
|
|
_reset_catalog_cache_for_imp85,
|
|
):
|
|
"""Fixture: non-VP contract with unregistered builder → u2 invariant raise."""
|
|
catalog = _load_fixture_catalog(_MISSING_BUILDER_FIXTURE)
|
|
with pytest.raises(CatalogInvariantError) as exc:
|
|
_check_catalog_builder_invariant(catalog)
|
|
msg = str(exc.value)
|
|
assert "imp85_u5_missing_builder_frame" in msg
|
|
assert "definitely_not_a_registered_builder_imp85_u5" in msg
|
|
|
|
|
|
def test_missing_builder_fixture_audit_reports_i3(tmp_path):
|
|
"""Fixture: non-VP contract with unregistered builder → audit I3 + I1.
|
|
|
|
The fixture frame's template_id has no partial on disk (tmp_path is empty),
|
|
so I1 fires as well. I3 is the primary assertion target; I1 surfacing is
|
|
expected and asserted to lock both audit paths together.
|
|
"""
|
|
violations = run_audit(_MISSING_BUILDER_FIXTURE, tmp_path)
|
|
joined = "\n".join(violations)
|
|
assert any(
|
|
v.startswith("I3 builder-unregistered:")
|
|
and "imp85_u5_missing_builder_frame" in v
|
|
for v in violations
|
|
), f"expected I3 builder-unregistered violation, got:\n{joined}"
|
|
assert any(
|
|
v.startswith("I1 partial-missing:")
|
|
and "imp85_u5_missing_builder_frame" in v
|
|
for v in violations
|
|
), f"expected I1 partial-missing violation, got:\n{joined}"
|
|
|
|
|
|
def test_undeclared_slot_fixture_audit_reports_i4(tmp_path):
|
|
"""Fixture: non-VP contract with valid builder but orphan generated key.
|
|
|
|
`items_with_role` + `array_root: orphan_array_root_imp85_u5` produces
|
|
`slot_payload.orphan_array_root_imp85_u5`. The temp partial below contains
|
|
`slot_payload.title` only (no bracket access), so I4 must fire on the
|
|
orphan array_root key.
|
|
"""
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
partial = partials_dir / "imp85_u5_undeclared_slot_frame.html"
|
|
partial.write_text(
|
|
"<div>{{ slot_payload.title }}</div>",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
violations = run_audit(_UNDECLARED_SLOT_FIXTURE, partials_dir)
|
|
joined = "\n".join(violations)
|
|
assert any(
|
|
v.startswith("I4 generated-key-orphan:")
|
|
and "imp85_u5_undeclared_slot_frame" in v
|
|
and "orphan_array_root_imp85_u5" in v
|
|
for v in violations
|
|
), f"expected I4 generated-key-orphan violation, got:\n{joined}"
|
|
|
|
|
|
def test_fixtures_with_visual_pending_true_are_skipped(
|
|
tmp_path, _reset_catalog_cache_for_imp85,
|
|
):
|
|
"""VP scope guard — flipping `visual_pending: true` on fixture frames must
|
|
silence both the boot invariant (u2) and the audit CLI (I1-I4)."""
|
|
missing = _load_fixture_catalog(_MISSING_BUILDER_FIXTURE)
|
|
undeclared = _load_fixture_catalog(_UNDECLARED_SLOT_FIXTURE)
|
|
for entry in (*missing.values(), *undeclared.values()):
|
|
entry["visual_pending"] = True
|
|
|
|
_check_catalog_builder_invariant(missing)
|
|
_check_catalog_builder_invariant(undeclared)
|
|
|
|
vp_yaml = tmp_path / "vp_only.yaml"
|
|
vp_yaml.write_text(yaml.safe_dump({**missing, **undeclared}), encoding="utf-8")
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
violations = run_audit(vp_yaml, partials_dir)
|
|
assert violations == [], (
|
|
f"VP frames must be silently skipped, got:\n - "
|
|
+ "\n - ".join(violations)
|
|
)
|