kyeongmin
cacc5b30db
- u1: BuilderMissingError(FitError) — narrow exception aligned with pipeline catch - u2: load_frame_contracts catalog invariant + VP skip + CatalogInvariantError - u3a: audit CLI I1~I3 (partial existence / declared builder / registry membership) - u3b: audit CLI I4 (slot_payload refs vs declared/generated payload keys) - u4: lookup_v4_candidates VP filter (lookup_v4_all_judgments raw telemetry untouched) - u5: catalog invariant regression coverage + temp non-VP failure fixtures - u6: mdx04 VP routing fixture tests (sw_dependency_four_problems excluded from live) - u7: tests/conftest.py env isolation + mdx03/mdx04/mdx05 subprocess smoke Targeted 74 PASS (12.31s). Full regression 1063 PASS (87.70s). Audit CLI clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
445 lines
15 KiB
Python
445 lines
15 KiB
Python
"""IMP-#85 u3b — Audit CLI invariant I4 (slot_payload ↔ builder generated keys).
|
|
|
|
Scope (Stage 2 lock):
|
|
I4 slot_payload refs — every key generated by the contract's builder must
|
|
appear as a `slot_payload.<key>` reference in the
|
|
partial. Direction A only (dead generated key).
|
|
Skipped when the partial uses dynamic bracket
|
|
access (`slot_payload[...]`).
|
|
|
|
`visual_pending: true` skipped (data-driven from catalog, matches u2/u3a
|
|
invariant scope; no hard-coded frame allow-list).
|
|
|
|
Out of scope (별 axis):
|
|
- V4 runtime VP filter (u4).
|
|
- Catalog regression coverage suite (u5).
|
|
- Implementing the 17 missing VP builders.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SCRIPT_PATH = REPO_ROOT / "scripts" / "audit_frame_invariants.py"
|
|
|
|
|
|
def _write_yaml(path: Path, payload: dict) -> Path:
|
|
path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def _run_cli(catalog: Path, partials: Path) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT_PATH),
|
|
"--catalog",
|
|
str(catalog),
|
|
"--partials-dir",
|
|
str(partials),
|
|
],
|
|
cwd=str(REPO_ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
|
|
def test_prod_catalog_audit_passes_i4():
|
|
"""Prod catalog + prod partials dir → no I4 violations on live contracts."""
|
|
from scripts.audit_frame_invariants import (
|
|
DEFAULT_CATALOG_PATH,
|
|
DEFAULT_PARTIALS_DIR,
|
|
check_i4_slot_payload_refs,
|
|
)
|
|
from src.phase_z2_mapper import PAYLOAD_BUILDERS
|
|
|
|
catalog = yaml.safe_load(
|
|
DEFAULT_CATALOG_PATH.read_text(encoding="utf-8")
|
|
) or {}
|
|
registered = set(PAYLOAD_BUILDERS.keys())
|
|
violations = check_i4_slot_payload_refs(
|
|
catalog, DEFAULT_PARTIALS_DIR, registered
|
|
)
|
|
assert violations == [], (
|
|
"Prod live contracts must satisfy I4 (every generated key is "
|
|
"referenced by the partial, or partial uses dynamic access). "
|
|
f"Got: {violations}"
|
|
)
|
|
|
|
|
|
def test_extract_static_slot_refs_finds_dot_access():
|
|
from scripts.audit_frame_invariants import extract_static_slot_refs
|
|
|
|
partial = (
|
|
"{{ slot_payload.title }}\n"
|
|
"{% if slot_payload.foo %}<b>{{ slot_payload.foo }}</b>{% endif %}\n"
|
|
"{% for x in slot_payload.bar %}{{ x }}{% endfor %}\n"
|
|
)
|
|
refs = extract_static_slot_refs(partial)
|
|
assert refs == {"title", "foo", "bar"}
|
|
|
|
|
|
def test_extract_static_slot_refs_ignores_dynamic_bracket():
|
|
from scripts.audit_frame_invariants import extract_static_slot_refs
|
|
|
|
partial = "{{ slot_payload['pill_' ~ n ~ '_label'] }}"
|
|
# Dynamic access does NOT contribute dot-access refs.
|
|
assert extract_static_slot_refs(partial) == set()
|
|
|
|
|
|
def test_partial_uses_dynamic_slot_access_detects_bracket():
|
|
from scripts.audit_frame_invariants import partial_uses_dynamic_slot_access
|
|
|
|
dynamic = "{{ slot_payload['pill_' ~ n ~ '_label'] }}"
|
|
static = "{{ slot_payload.title }} and {{ slot_payload.body }}"
|
|
assert partial_uses_dynamic_slot_access(dynamic) is True
|
|
assert partial_uses_dynamic_slot_access(static) is False
|
|
|
|
|
|
def test_expected_keys_quadrant_flat_slots_default_pattern():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item",
|
|
"pad_to": 4,
|
|
"label_key_pattern": "quadrant_{n}_label",
|
|
"body_key_pattern": "quadrant_{n}_body",
|
|
},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert "title" in keys
|
|
for n in range(1, 5):
|
|
assert f"quadrant_{n}_label" in keys
|
|
assert f"quadrant_{n}_body" in keys
|
|
|
|
|
|
def test_expected_keys_quadrant_flat_slots_custom_pattern():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item",
|
|
"pad_to": 3,
|
|
"label_key_pattern": "category_{n}_label",
|
|
"body_key_pattern": "category_{n}_body",
|
|
},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert keys == {
|
|
"title",
|
|
"category_1_label", "category_2_label", "category_3_label",
|
|
"category_1_body", "category_2_body", "category_3_body",
|
|
}
|
|
|
|
|
|
def test_expected_keys_cycle_intersect_3():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "cycle_intersect_3",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item",
|
|
"pad_to": 3,
|
|
"label_key_pattern": "circle_{n}_label",
|
|
},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert keys == {
|
|
"title", "circle_1_label", "circle_2_label", "circle_3_label",
|
|
"intersection",
|
|
}
|
|
|
|
|
|
def test_expected_keys_compare_table_2col():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "compare_table_2col",
|
|
"builder_options": {"item_parser": "compare_row_2col_item"},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert keys == {"title", "col_a_label", "col_b_label", "rows"}
|
|
|
|
|
|
def test_expected_keys_paired_rows_4x2_slots():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "paired_rows_4x2_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item",
|
|
"label_key_pattern": "row_{r}_{side}_label",
|
|
"body_key_pattern": "row_{r}_{side}_body",
|
|
"rows": 4,
|
|
"sides": ["left", "right"],
|
|
},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert "title" in keys
|
|
for r in range(1, 5):
|
|
for side in ("left", "right"):
|
|
assert f"row_{r}_{side}_label" in keys
|
|
assert f"row_{r}_{side}_body" in keys
|
|
|
|
|
|
def test_expected_keys_process_product_pair():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "process_product_pair",
|
|
"builder_options": {
|
|
"pad_sections_to": 3,
|
|
"columns": [
|
|
{"title_to": "banner_left", "body_to": "process",
|
|
"body_parser": "column_with_transform"},
|
|
{"title_to": "banner_right", "body_to": "product",
|
|
"body_parser": "column_with_transform"},
|
|
],
|
|
},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert keys == {"title", "banner_left", "process", "banner_right", "product"}
|
|
|
|
|
|
def test_expected_keys_items_with_role():
|
|
from scripts.audit_frame_invariants import expected_payload_keys
|
|
|
|
contract = {
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "items_with_role",
|
|
"builder_options": {
|
|
"item_parser": "pillar_item",
|
|
"array_root": "pillars",
|
|
},
|
|
}
|
|
}
|
|
keys = expected_payload_keys(contract)
|
|
assert keys == {"title", "pillars"}
|
|
|
|
|
|
def test_i4_dead_generated_key_flagged(tmp_path):
|
|
"""Builder produces key X, partial doesn't reference it → I4 violation."""
|
|
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
|
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
# Partial only references `title` — missing category_2_label / _body etc.
|
|
(partials_dir / "drift_frame.html").write_text(
|
|
"<div>{{ slot_payload.title }}</div>"
|
|
"<div>{{ slot_payload.category_1_label }}</div>"
|
|
"<div>{{ slot_payload.category_1_body }}</div>",
|
|
encoding="utf-8",
|
|
)
|
|
catalog = {
|
|
"drift_frame": {
|
|
"template_id": "drift_frame",
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item",
|
|
"pad_to": 2,
|
|
"label_key_pattern": "category_{n}_label",
|
|
"body_key_pattern": "category_{n}_body",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
violations = check_i4_slot_payload_refs(
|
|
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
|
)
|
|
msgs = "\n".join(violations)
|
|
assert "I4 generated-key-orphan" in msgs
|
|
assert "drift_frame" in msgs
|
|
assert "category_2_label" in msgs
|
|
assert "category_2_body" in msgs
|
|
# category_1 keys ARE referenced — must NOT be flagged.
|
|
assert "slot_payload.category_1_label." not in msgs
|
|
assert "slot_payload.category_1_body." not in msgs
|
|
|
|
|
|
def test_i4_skips_partial_with_dynamic_bracket_access(tmp_path):
|
|
"""Dynamic bracket access in partial → I4 skipped (cannot resolve statically)."""
|
|
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
|
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
(partials_dir / "dynamic_frame.html").write_text(
|
|
"{{ slot_payload.title }}\n"
|
|
"{% for n in range(1, 6) %}"
|
|
"{{ slot_payload['pill_' ~ n ~ '_label'] }}"
|
|
"{{ slot_payload['pill_' ~ n ~ '_body'] }}"
|
|
"{% endfor %}",
|
|
encoding="utf-8",
|
|
)
|
|
catalog = {
|
|
"dynamic_frame": {
|
|
"template_id": "dynamic_frame",
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item",
|
|
"pad_to": 5,
|
|
"label_key_pattern": "pill_{n}_label",
|
|
"body_key_pattern": "pill_{n}_body",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
violations = check_i4_slot_payload_refs(
|
|
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
|
)
|
|
assert violations == [], (
|
|
"Dynamic bracket access must suppress I4 (cannot resolve statically); "
|
|
f"got: {violations}"
|
|
)
|
|
|
|
|
|
def test_i4_skips_visual_pending(tmp_path):
|
|
"""VP contract with drift → I4 skip (no violation)."""
|
|
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
|
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
(partials_dir / "vp_frame.html").write_text(
|
|
"<div>nothing</div>", encoding="utf-8"
|
|
)
|
|
catalog = {
|
|
"vp_frame": {
|
|
"template_id": "vp_frame",
|
|
"visual_pending": True,
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item", "pad_to": 4,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
violations = check_i4_slot_payload_refs(
|
|
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
|
)
|
|
assert violations == []
|
|
|
|
|
|
def test_i4_skips_unregistered_builder(tmp_path):
|
|
"""Unregistered builder (already an I3 hit) → I4 silent on same contract."""
|
|
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
|
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
(partials_dir / "ghost_frame.html").write_text(
|
|
"{{ slot_payload.title }}", encoding="utf-8"
|
|
)
|
|
catalog = {
|
|
"ghost_frame": {
|
|
"template_id": "ghost_frame",
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "ghost_builder_not_in_registry",
|
|
},
|
|
},
|
|
}
|
|
violations = check_i4_slot_payload_refs(
|
|
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
|
)
|
|
assert violations == [], (
|
|
"Unregistered builder is already flagged by I3 — I4 must stay silent "
|
|
f"on the same contract; got: {violations}"
|
|
)
|
|
|
|
|
|
def test_i4_skips_missing_partial(tmp_path):
|
|
"""Missing partial (already I1 hit) → I4 silent on same contract."""
|
|
from scripts.audit_frame_invariants import check_i4_slot_payload_refs
|
|
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
# No partial file written.
|
|
catalog = {
|
|
"missing_partial_frame": {
|
|
"template_id": "missing_partial_frame",
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item", "pad_to": 4,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
violations = check_i4_slot_payload_refs(
|
|
catalog, partials_dir, registered_builders={"quadrant_flat_slots"}
|
|
)
|
|
assert violations == []
|
|
|
|
|
|
def test_cli_pass_on_prod_paths(tmp_path):
|
|
"""End-to-end CLI on prod paths reports PASS with I1-I4 wording."""
|
|
result = subprocess.run(
|
|
[sys.executable, str(SCRIPT_PATH)],
|
|
cwd=str(REPO_ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
assert "PASS (I1-I4 clean" in result.stdout
|
|
|
|
|
|
def test_cli_fail_on_synthetic_i4_drift(tmp_path):
|
|
"""CLI exits 1 + emits I4 violation when a non-VP contract has dead keys."""
|
|
partials_dir = tmp_path / "families"
|
|
partials_dir.mkdir()
|
|
(partials_dir / "drift_frame.html").write_text(
|
|
"{{ slot_payload.title }}", encoding="utf-8"
|
|
)
|
|
catalog_path = _write_yaml(
|
|
tmp_path / "frame_contracts.yaml",
|
|
{
|
|
"drift_frame": {
|
|
"template_id": "drift_frame",
|
|
"payload": {
|
|
"title": {"source": "section.title"},
|
|
"builder": "quadrant_flat_slots",
|
|
"builder_options": {
|
|
"item_parser": "quadrant_item", "pad_to": 2,
|
|
"label_key_pattern": "category_{n}_label",
|
|
"body_key_pattern": "category_{n}_body",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
)
|
|
result = _run_cli(catalog_path, partials_dir)
|
|
assert result.returncode == 1, result.stdout + result.stderr
|
|
assert "I4 generated-key-orphan" in result.stdout
|
|
assert "category_1_label" in result.stdout
|