KyeongminandClaude Fable 5
4d7c401779
Multi-MDX Regression (IMP-91) / multi-mdx-regression (push) Failing after 9m49s
- tests/integration/scripts/regenerate_snapshots.py: 스냅샷 일괄 재생성 도구 - 통합 스냅샷 9종 갱신, 회귀/유닛 테스트 7건 보강 - ISSUE_DRAFTS_2026-07-02.md: 7/2 전수 검토 이슈 초안 보존 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
258 lines
10 KiB
Python
258 lines
10 KiB
Python
"""IMP-91 스냅샷 재생성 절차 (GitHub issue #29).
|
|
|
|
structural.json 의 _doc 계약: "Update only when an intentional pipeline
|
|
change moves the observed value" — 의도적 변경 후 이 스크립트로 갱신한다.
|
|
테스트 모듈의 실행 함수(run_pipeline_for_snapshot — AI OFF 강제)와 shape
|
|
헬퍼(_slot_payload_zone_shape 등)를 그대로 import 해 추출 로직 표류를 방지.
|
|
|
|
실행: python -m tests.integration.scripts.regenerate_snapshots
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
from tests.integration.test_multi_mdx_regression import ( # noqa: E402
|
|
MDX_SET,
|
|
SNAPSHOTS_DIR,
|
|
_AI_UNIT_KEYS,
|
|
_SLIDE_ROOT_RE,
|
|
_TITLE_RE,
|
|
_extract_html_zone_topology,
|
|
_layout_zone_shape,
|
|
_slot_payload_zone_shape,
|
|
run_pipeline_for_snapshot,
|
|
)
|
|
|
|
|
|
def _j(run_dir: Path, name: str) -> dict:
|
|
return json.loads((run_dir / "steps" / name).read_text(encoding="utf-8"))
|
|
|
|
|
|
def extract_structural(run_dir: Path) -> dict:
|
|
status = _j(run_dir, "step20_slide_status.json")["data"]
|
|
frame_sel = _j(run_dir, "step09_frame_selection.json")["data"]
|
|
zones = frame_sel.get("per_zone", [])
|
|
return {
|
|
"overall": status.get("overall"),
|
|
"zone_count": len(zones),
|
|
"zones": [
|
|
{"position": z.get("position"),
|
|
"selected_template_id": z.get("selected_template_id")}
|
|
for z in zones
|
|
],
|
|
}
|
|
|
|
|
|
def extract_visual(run_dir: Path) -> dict:
|
|
visual = _j(run_dir, "step14_visual_check.json")["data"]
|
|
return {
|
|
"slide_overflowed": visual.get("slide", {}).get("overflowed"),
|
|
"slide_body_overflowed": visual.get("slide_body", {}).get("overflowed"),
|
|
"passed": visual.get("passed"),
|
|
"zones": [
|
|
{
|
|
"position": z.get("position"),
|
|
"template_id": z.get("template_id"),
|
|
"overflowed": z.get("overflowed"),
|
|
"clipped_inner_count": len(z.get("clipped_inner") or []),
|
|
}
|
|
for z in visual.get("zones", [])
|
|
],
|
|
}
|
|
|
|
|
|
def extract_coverage(run_dir: Path) -> dict:
|
|
status = _j(run_dir, "step20_slide_status.json")["data"]
|
|
return {
|
|
"rendered": status.get("rendered"),
|
|
"visual_check_passed": status.get("visual_check_passed"),
|
|
"full_mdx_coverage": status.get("full_mdx_coverage"),
|
|
"aligned_section_ids": sorted(status.get("aligned_section_ids") or []),
|
|
"covered_section_ids": sorted(status.get("covered_section_ids") or []),
|
|
"filtered_section_ids": sorted(status.get("filtered_section_ids") or []),
|
|
}
|
|
|
|
|
|
def extract_normalize(run_dir: Path) -> dict:
|
|
raw = _j(run_dir, "step02_normalized.json")
|
|
d = raw["data"]
|
|
diag = d.get("stage0_adapter_diagnostics", {}) or {}
|
|
assets = d.get("stage0_normalized_assets", {}) or {}
|
|
return {
|
|
"step_num": raw.get("step_num"),
|
|
"step_status": raw.get("step_status"),
|
|
"pipeline_path_connected": raw.get("pipeline_path_connected"),
|
|
"sections_count": d.get("sections_count"),
|
|
"section_ids": [s.get("section_id") for s in d.get("sections", [])],
|
|
"orphans_count": len(d.get("orphans") or []),
|
|
"details_count": len(d.get("details") or []),
|
|
"adapter_enabled": diag.get("enabled"),
|
|
"adapter_used": diag.get("used"),
|
|
"assets_popups_count": len(assets.get("popups") or []),
|
|
"assets_images_count": len(assets.get("images") or []),
|
|
"assets_tables_count": len(assets.get("tables") or []),
|
|
"slide_title_nonempty": bool(d.get("slide_title")),
|
|
"slide_footer_nonempty": bool(d.get("slide_footer")),
|
|
}
|
|
|
|
|
|
def extract_v4_ranking(run_dir: Path) -> dict:
|
|
data = _j(run_dir, "step05_v4_evidence.json")["data"]
|
|
return {
|
|
"v4_source": str(data.get("v4_source") or "").replace("\\", "/"),
|
|
"aligned_section_ids": data.get("aligned_section_ids"),
|
|
"sections": [
|
|
{
|
|
"section_id": ev.get("section_id"),
|
|
"candidate_status": ev.get("candidate_status"),
|
|
"candidates": [
|
|
{
|
|
"template_id": c.get("template_id"),
|
|
"label": c.get("label"),
|
|
"confidence": c.get("confidence"),
|
|
}
|
|
for c in (ev.get("v4_candidates") or [])
|
|
],
|
|
}
|
|
for ev in (data.get("evidence_per_section") or [])
|
|
],
|
|
}
|
|
|
|
|
|
def extract_ai_classifier(run_dir: Path) -> dict:
|
|
ai = _j(run_dir, "step12_ai_repair.json")["data"]
|
|
fit = _j(run_dir, "step15_fit_classification.json")["data"]
|
|
router = _j(run_dir, "step16_router_decision.json")["data"]
|
|
failure = _j(run_dir, "step18_failure_classification.json")["data"]
|
|
units = [{k: u.get(k) for k in _AI_UNIT_KEYS} for u in (ai.get("per_unit") or [])]
|
|
return {
|
|
"units": units,
|
|
"coverage_invariant_status": (ai.get("coverage_invariant") or {}).get("status"),
|
|
"fit_visual_check_passed": fit.get("visual_check_passed"),
|
|
"fit_classifications_count": len(fit.get("classifications") or []),
|
|
"fit_categories_seen": fit.get("categories_seen") or [],
|
|
"router_active": router.get("router_active"),
|
|
"router_routed_count": router.get("routed_count"),
|
|
"router_v4_fallback_used_count": (router.get("v4_fallback_summary") or {}).get("fallback_used_count"),
|
|
"failure_type": failure.get("failure_type"),
|
|
}
|
|
|
|
|
|
def extract_layout(run_dir: Path) -> dict:
|
|
s7 = _j(run_dir, "step07_layout.json")
|
|
s8 = _j(run_dir, "step08_zone_region_ratios.json")
|
|
d7 = s7.get("data") or {}
|
|
d8 = s8.get("data") or {}
|
|
css = d7.get("layout_css") or {}
|
|
return {
|
|
"step7_step_status": s7.get("step_status"),
|
|
"step7_pipeline_path_connected": s7.get("pipeline_path_connected"),
|
|
"layout_preset": d7.get("layout_preset"),
|
|
"auto_layout_preset": d7.get("auto_layout_preset"),
|
|
"layout_override_applied": d7.get("layout_override_applied"),
|
|
"zones_count": d7.get("zones_count"),
|
|
"unit_count": d7.get("unit_count"),
|
|
"layout_candidates": d7.get("layout_candidates") or [],
|
|
"computation": css.get("computation"),
|
|
"dynamic_rows": css.get("dynamic_rows"),
|
|
"dynamic_cols": css.get("dynamic_cols"),
|
|
"heights_px": css.get("heights_px"),
|
|
"widths_px": css.get("widths_px"),
|
|
"ratios": css.get("ratios"),
|
|
"width_ratios": css.get("width_ratios"),
|
|
"step8_step_status": s8.get("step_status"),
|
|
"step8_pipeline_path_connected": s8.get("pipeline_path_connected"),
|
|
"zone_heights_px_planned": d8.get("zone_heights_px_planned"),
|
|
"zone_widths_px_planned": d8.get("zone_widths_px_planned"),
|
|
"zone_col_ratios_planned": d8.get("zone_col_ratios_planned"),
|
|
"per_zone_layout_shape": [
|
|
_layout_zone_shape(z) for z in (d8.get("per_zone_plan") or [])
|
|
],
|
|
}
|
|
|
|
|
|
def extract_slot_payload(run_dir: Path) -> list:
|
|
raw = _j(run_dir, "step12_slot_payload.json")
|
|
return [_slot_payload_zone_shape(z) for z in raw["data"].get("per_zone") or []]
|
|
|
|
|
|
def extract_final_html(run_dir: Path) -> dict:
|
|
raw13 = _j(run_dir, "step13_render.json")
|
|
d13 = raw13.get("data") or {}
|
|
ri = d13.get("render_inputs") or {}
|
|
final_path = run_dir / "final.html"
|
|
html = final_path.read_text(encoding="utf-8")
|
|
title_match = _TITLE_RE.search(html)
|
|
html_title = title_match.group(1).strip() if title_match else ""
|
|
html_topology = _extract_html_zone_topology(html)
|
|
return {
|
|
"step13_status": raw13.get("step_status"),
|
|
"step13_pipeline_path_connected": raw13.get("pipeline_path_connected"),
|
|
"render_inputs_zones_count": ri.get("zones_count"),
|
|
"render_inputs_layout_preset": ri.get("layout_preset"),
|
|
"render_inputs_slide_title_nonempty": bool((ri.get("slide_title") or "").strip()),
|
|
"render_inputs_slide_footer_nonempty": bool((ri.get("slide_footer") or "").strip()),
|
|
"html_title_matches_render_input": html_title == (ri.get("slide_title") or "").strip(),
|
|
"html_slide_root_count": len(_SLIDE_ROOT_RE.findall(html)),
|
|
"html_slide_footer_present": '<div class="slide-footer">' in html,
|
|
"html_zone_count": len(html_topology),
|
|
"html_zone_topology": html_topology,
|
|
"final_html_size_matches_step13_reported": (
|
|
final_path.stat().st_size == d13.get("final_html_size_bytes")
|
|
),
|
|
}
|
|
|
|
|
|
EXTRACTORS = {
|
|
"structural.json": extract_structural,
|
|
"visual.json": extract_visual,
|
|
"coverage.json": extract_coverage,
|
|
"normalize.json": extract_normalize,
|
|
"v4_ranking.json": extract_v4_ranking,
|
|
"ai_classifier.json": extract_ai_classifier,
|
|
"layout.json": extract_layout,
|
|
"slot_payload.json": extract_slot_payload,
|
|
"final_html.json": extract_final_html,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
runs = {}
|
|
for mdx_id in MDX_SET:
|
|
run_id = f"imp91_regen_{mdx_id}_{uuid.uuid4().hex[:8]}"
|
|
print(f"[regen] running {mdx_id}.mdx (AI OFF) ...")
|
|
run = run_pipeline_for_snapshot(mdx_id, run_id)
|
|
if run.returncode != 0:
|
|
raise RuntimeError(
|
|
f"{mdx_id}.mdx run failed rc={run.returncode}: {run.stderr[-400:]}"
|
|
)
|
|
runs[mdx_id] = run
|
|
|
|
for name, extractor in EXTRACTORS.items():
|
|
path = SNAPSHOTS_DIR / name
|
|
existing = json.loads(path.read_text(encoding="utf-8"))
|
|
new_doc = {}
|
|
if "_doc" in existing:
|
|
new_doc["_doc"] = existing["_doc"]
|
|
for mdx_id in MDX_SET:
|
|
new_doc[mdx_id] = extractor(runs[mdx_id].run_dir)
|
|
path.write_text(
|
|
json.dumps(new_doc, ensure_ascii=False, indent=1) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
changed = [m for m in MDX_SET if existing.get(m) != new_doc[m]]
|
|
print(f"[regen] {name}: 갱신 mdx {changed or '없음'}")
|
|
|
|
print("완료 — pytest tests/integration 으로 검증하세요.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|