feat(#85): IMP catalog builder invariant + VP runtime gate (u1~u7)
- u1: BuilderMissingError(FitError) — narrow exception aligned with pipeline catch - u2: load_frame_contracts catalog invariant + VP skip + CatalogInvariantError - u3a: audit CLI I1~I3 (partial existence / declared builder / registry membership) - u3b: audit CLI I4 (slot_payload refs vs declared/generated payload keys) - u4: lookup_v4_candidates VP filter (lookup_v4_all_judgments raw telemetry untouched) - u5: catalog invariant regression coverage + temp non-VP failure fixtures - u6: mdx04 VP routing fixture tests (sw_dependency_four_problems excluded from live) - u7: tests/conftest.py env isolation + mdx03/mdx04/mdx05 subprocess smoke Targeted 74 PASS (12.31s). Full regression 1063 PASS (87.70s). Audit CLI clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
444
tests/test_audit_frame_invariants_i4.py
Normal file
444
tests/test_audit_frame_invariants_i4.py
Normal file
@@ -0,0 +1,444 @@
|
||||
"""IMP-#85 u3b — Audit CLI invariant I4 (slot_payload ↔ builder generated keys).
|
||||
|
||||
Scope (Stage 2 lock):
|
||||
I4 slot_payload refs — every key generated by the contract's builder must
|
||||
appear as a `slot_payload.<key>` reference in the
|
||||
partial. Direction A only (dead generated key).
|
||||
Skipped when the partial uses dynamic bracket
|
||||
access (`slot_payload[...]`).
|
||||
|
||||
`visual_pending: true` skipped (data-driven from catalog, matches u2/u3a
|
||||
invariant scope; no hard-coded frame allow-list).
|
||||
|
||||
Out of scope (별 axis):
|
||||
- V4 runtime VP filter (u4).
|
||||
- Catalog regression coverage suite (u5).
|
||||
- Implementing the 17 missing VP builders.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_PATH = REPO_ROOT / "scripts" / "audit_frame_invariants.py"
|
||||
|
||||
|
||||
def _write_yaml(path: Path, payload: dict) -> Path:
|
||||
path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _run_cli(catalog: Path, partials: Path) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT_PATH),
|
||||
"--catalog",
|
||||
str(catalog),
|
||||
"--partials-dir",
|
||||
str(partials),
|
||||
],
|
||||
cwd=str(REPO_ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_prod_catalog_audit_passes_i4():
|
||||
"""Prod catalog + prod partials dir → no I4 violations on live contracts."""
|
||||
from scripts.audit_frame_invariants import (
|
||||
DEFAULT_CATALOG_PATH,
|
||||
DEFAULT_PARTIALS_DIR,
|
||||
check_i4_slot_payload_refs,
|
||||
)
|
||||
from src.phase_z2_mapper import PAYLOAD_BUILDERS
|
||||
|
||||
catalog = yaml.safe_load(
|
||||
DEFAULT_CATALOG_PATH.read_text(encoding="utf-8")
|
||||
) or {}
|
||||
registered = set(PAYLOAD_BUILDERS.keys())
|
||||
violations = check_i4_slot_payload_refs(
|
||||
catalog, DEFAULT_PARTIALS_DIR, registered
|
||||
)
|
||||
assert violations == [], (
|
||||
"Prod live contracts must satisfy I4 (every generated key is "
|
||||
"referenced by the partial, or partial uses dynamic access). "
|
||||
f"Got: {violations}"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_static_slot_refs_finds_dot_access():
|
||||
from scripts.audit_frame_invariants import extract_static_slot_refs
|
||||
|
||||
partial = (
|
||||
"{{ slot_payload.title }}\n"
|
||||
"{% if slot_payload.foo %}<b>{{ slot_payload.foo }}</b>{% endif %}\n"
|
||||
"{% for x in slot_payload.bar %}{{ x }}{% endfor %}\n"
|
||||
)
|
||||
refs = extract_static_slot_refs(partial)
|
||||
assert refs == {"title", "foo", "bar"}
|
||||
|
||||
|
||||
def test_extract_static_slot_refs_ignores_dynamic_bracket():
|
||||
from scripts.audit_frame_invariants import extract_static_slot_refs
|
||||
|
||||
partial = "{{ slot_payload['pill_' ~ n ~ '_label'] }}"
|
||||
# Dynamic access does NOT contribute dot-access refs.
|
||||
assert extract_static_slot_refs(partial) == set()
|
||||
|
||||
|
||||
def test_partial_uses_dynamic_slot_access_detects_bracket():
|
||||
from scripts.audit_frame_invariants import partial_uses_dynamic_slot_access
|
||||
|
||||
dynamic = "{{ slot_payload['pill_' ~ n ~ '_label'] }}"
|
||||
static = "{{ slot_payload.title }} and {{ slot_payload.body }}"
|
||||
assert partial_uses_dynamic_slot_access(dynamic) is True
|
||||
assert partial_uses_dynamic_slot_access(static) is False
|
||||
|
||||
|
||||
def test_expected_keys_quadrant_flat_slots_default_pattern():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item",
|
||||
"pad_to": 4,
|
||||
"label_key_pattern": "quadrant_{n}_label",
|
||||
"body_key_pattern": "quadrant_{n}_body",
|
||||
},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert "title" in keys
|
||||
for n in range(1, 5):
|
||||
assert f"quadrant_{n}_label" in keys
|
||||
assert f"quadrant_{n}_body" in keys
|
||||
|
||||
|
||||
def test_expected_keys_quadrant_flat_slots_custom_pattern():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item",
|
||||
"pad_to": 3,
|
||||
"label_key_pattern": "category_{n}_label",
|
||||
"body_key_pattern": "category_{n}_body",
|
||||
},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert keys == {
|
||||
"title",
|
||||
"category_1_label", "category_2_label", "category_3_label",
|
||||
"category_1_body", "category_2_body", "category_3_body",
|
||||
}
|
||||
|
||||
|
||||
def test_expected_keys_cycle_intersect_3():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "cycle_intersect_3",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item",
|
||||
"pad_to": 3,
|
||||
"label_key_pattern": "circle_{n}_label",
|
||||
},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert keys == {
|
||||
"title", "circle_1_label", "circle_2_label", "circle_3_label",
|
||||
"intersection",
|
||||
}
|
||||
|
||||
|
||||
def test_expected_keys_compare_table_2col():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "compare_table_2col",
|
||||
"builder_options": {"item_parser": "compare_row_2col_item"},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert keys == {"title", "col_a_label", "col_b_label", "rows"}
|
||||
|
||||
|
||||
def test_expected_keys_paired_rows_4x2_slots():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "paired_rows_4x2_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item",
|
||||
"label_key_pattern": "row_{r}_{side}_label",
|
||||
"body_key_pattern": "row_{r}_{side}_body",
|
||||
"rows": 4,
|
||||
"sides": ["left", "right"],
|
||||
},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert "title" in keys
|
||||
for r in range(1, 5):
|
||||
for side in ("left", "right"):
|
||||
assert f"row_{r}_{side}_label" in keys
|
||||
assert f"row_{r}_{side}_body" in keys
|
||||
|
||||
|
||||
def test_expected_keys_process_product_pair():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "process_product_pair",
|
||||
"builder_options": {
|
||||
"pad_sections_to": 3,
|
||||
"columns": [
|
||||
{"title_to": "banner_left", "body_to": "process",
|
||||
"body_parser": "column_with_transform"},
|
||||
{"title_to": "banner_right", "body_to": "product",
|
||||
"body_parser": "column_with_transform"},
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert keys == {"title", "banner_left", "process", "banner_right", "product"}
|
||||
|
||||
|
||||
def test_expected_keys_items_with_role():
|
||||
from scripts.audit_frame_invariants import expected_payload_keys
|
||||
|
||||
contract = {
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "items_with_role",
|
||||
"builder_options": {
|
||||
"item_parser": "pillar_item",
|
||||
"array_root": "pillars",
|
||||
},
|
||||
}
|
||||
}
|
||||
keys = expected_payload_keys(contract)
|
||||
assert keys == {"title", "pillars"}
|
||||
|
||||
|
||||
def test_i4_dead_generated_key_flagged(tmp_path):
|
||||
"""Builder produces key X, partial doesn't reference it → I4 violation."""
|
||||
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
||||
|
||||
partials_dir = tmp_path / "families"
|
||||
partials_dir.mkdir()
|
||||
# Partial only references `title` — missing category_2_label / _body etc.
|
||||
(partials_dir / "drift_frame.html").write_text(
|
||||
"<div>{{ slot_payload.title }}</div>"
|
||||
"<div>{{ slot_payload.category_1_label }}</div>"
|
||||
"<div>{{ slot_payload.category_1_body }}</div>",
|
||||
encoding="utf-8",
|
||||
)
|
||||
catalog = {
|
||||
"drift_frame": {
|
||||
"template_id": "drift_frame",
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item",
|
||||
"pad_to": 2,
|
||||
"label_key_pattern": "category_{n}_label",
|
||||
"body_key_pattern": "category_{n}_body",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
violations = check_i4_slot_payload_refs(
|
||||
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
||||
)
|
||||
msgs = "\n".join(violations)
|
||||
assert "I4 generated-key-orphan" in msgs
|
||||
assert "drift_frame" in msgs
|
||||
assert "category_2_label" in msgs
|
||||
assert "category_2_body" in msgs
|
||||
# category_1 keys ARE referenced — must NOT be flagged.
|
||||
assert "slot_payload.category_1_label." not in msgs
|
||||
assert "slot_payload.category_1_body." not in msgs
|
||||
|
||||
|
||||
def test_i4_skips_partial_with_dynamic_bracket_access(tmp_path):
|
||||
"""Dynamic bracket access in partial → I4 skipped (cannot resolve statically)."""
|
||||
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
||||
|
||||
partials_dir = tmp_path / "families"
|
||||
partials_dir.mkdir()
|
||||
(partials_dir / "dynamic_frame.html").write_text(
|
||||
"{{ slot_payload.title }}\n"
|
||||
"{% for n in range(1, 6) %}"
|
||||
"{{ slot_payload['pill_' ~ n ~ '_label'] }}"
|
||||
"{{ slot_payload['pill_' ~ n ~ '_body'] }}"
|
||||
"{% endfor %}",
|
||||
encoding="utf-8",
|
||||
)
|
||||
catalog = {
|
||||
"dynamic_frame": {
|
||||
"template_id": "dynamic_frame",
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item",
|
||||
"pad_to": 5,
|
||||
"label_key_pattern": "pill_{n}_label",
|
||||
"body_key_pattern": "pill_{n}_body",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
violations = check_i4_slot_payload_refs(
|
||||
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
||||
)
|
||||
assert violations == [], (
|
||||
"Dynamic bracket access must suppress I4 (cannot resolve statically); "
|
||||
f"got: {violations}"
|
||||
)
|
||||
|
||||
|
||||
def test_i4_skips_visual_pending(tmp_path):
|
||||
"""VP contract with drift → I4 skip (no violation)."""
|
||||
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
||||
|
||||
partials_dir = tmp_path / "families"
|
||||
partials_dir.mkdir()
|
||||
(partials_dir / "vp_frame.html").write_text(
|
||||
"<div>nothing</div>", encoding="utf-8"
|
||||
)
|
||||
catalog = {
|
||||
"vp_frame": {
|
||||
"template_id": "vp_frame",
|
||||
"visual_pending": True,
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item", "pad_to": 4,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
violations = check_i4_slot_payload_refs(
|
||||
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
||||
)
|
||||
assert violations == []
|
||||
|
||||
|
||||
def test_i4_skips_unregistered_builder(tmp_path):
|
||||
"""Unregistered builder (already an I3 hit) → I4 silent on same contract."""
|
||||
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
||||
|
||||
partials_dir = tmp_path / "families"
|
||||
partials_dir.mkdir()
|
||||
(partials_dir / "ghost_frame.html").write_text(
|
||||
"{{ slot_payload.title }}", encoding="utf-8"
|
||||
)
|
||||
catalog = {
|
||||
"ghost_frame": {
|
||||
"template_id": "ghost_frame",
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "ghost_builder_not_in_registry",
|
||||
},
|
||||
},
|
||||
}
|
||||
violations = check_i4_slot_payload_refs(
|
||||
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
||||
)
|
||||
assert violations == [], (
|
||||
"Unregistered builder is already flagged by I3 — I4 must stay silent "
|
||||
f"on the same contract; got: {violations}"
|
||||
)
|
||||
|
||||
|
||||
def test_i4_skips_missing_partial(tmp_path):
|
||||
"""Missing partial (already I1 hit) → I4 silent on same contract."""
|
||||
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
||||
|
||||
partials_dir = tmp_path / "families"
|
||||
partials_dir.mkdir()
|
||||
# No partial file written.
|
||||
catalog = {
|
||||
"missing_partial_frame": {
|
||||
"template_id": "missing_partial_frame",
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item", "pad_to": 4,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
violations = check_i4_slot_payload_refs(
|
||||
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
||||
)
|
||||
assert violations == []
|
||||
|
||||
|
||||
def test_cli_pass_on_prod_paths(tmp_path):
|
||||
"""End-to-end CLI on prod paths reports PASS with I1-I4 wording."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPT_PATH)],
|
||||
cwd=str(REPO_ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert "PASS (I1-I4 clean" in result.stdout
|
||||
|
||||
|
||||
def test_cli_fail_on_synthetic_i4_drift(tmp_path):
|
||||
"""CLI exits 1 + emits I4 violation when a non-VP contract has dead keys."""
|
||||
partials_dir = tmp_path / "families"
|
||||
partials_dir.mkdir()
|
||||
(partials_dir / "drift_frame.html").write_text(
|
||||
"{{ slot_payload.title }}", encoding="utf-8"
|
||||
)
|
||||
catalog_path = _write_yaml(
|
||||
tmp_path / "frame_contracts.yaml",
|
||||
{
|
||||
"drift_frame": {
|
||||
"template_id": "drift_frame",
|
||||
"payload": {
|
||||
"title": {"source": "section.title"},
|
||||
"builder": "quadrant_flat_slots",
|
||||
"builder_options": {
|
||||
"item_parser": "quadrant_item", "pad_to": 2,
|
||||
"label_key_pattern": "category_{n}_label",
|
||||
"body_key_pattern": "category_{n}_body",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
result = _run_cli(catalog_path, partials_dir)
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
assert "I4 generated-key-orphan" in result.stdout
|
||||
assert "category_1_label" in result.stdout
|
||||
Reference in New Issue
Block a user