wip: phase_z2 evidence 파이프라인 + matching 실험(phase2~26) + 프론트 trace 패널 진행분 스냅샷

- src: phase_z2 composition/mapper/pipeline/placement_planner/retry, ai_fallback(prompts/schema/validate), mdx_text_atoms 신규 - Front: PipelineTracePanel 신규, FramePanel/SlideCanvas/Home/designAgentApi 등 갱신 + 테스트 4종 추가 - templates/phase_z2: catalog(component_expansion_registry, node_slot_mapping 신규), frames, families, slide_base 갱신 - tests/matching: phase2~26 매칭 실험 스크립트·리포트·온톨로지 전체 (미커밋 진행분) - tests: b4_v4 evidence, task5~28.5 시리즈, regression(imp95 baseline) 등 신규 테스트 대량 추가 - docs/reference: MDX 구조 인벤토리, MDX→Frame 구조 계약 문서 - scripts: mdx 계약/parity/coverage/viewport 체크, gitea comment, run sync 유틸 - .gitignore: tmp*.json, chromedriver, .orchestrator, *.pkl, Front_test* 등 임시/스냅샷 제외 미완성 작업의 보존용 스냅샷 커밋 (2026-07-02) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-02 17:03:42 +09:00
parent 97b7833a1b
commit b836e79ee1
527 changed files with 673036 additions and 717 deletions
@@ -0,0 +1,203 @@
+"""8개 매칭 방법 실행 + 비교 매트릭스 리포트 생성"""
+import sys
+import os
+import yaml
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+from common import (
+    load_ground_truth, load_figma_texts, load_mdx_sections,
+    evaluate_ranking, aggregate_metrics
+)
+from methods import (
+    method_tfidf, method_bm25, method_char_ngram,
+    method_kiwi_bm25, method_structural,
+    method_ai_metadata_matcher, method_weighted, method_hard_filter,
+)
+
+
+def run_method_standard(name, fn, mdx, figma, gt_list):
+    gt_by_id = {g["id"]: g for g in gt_list}
+    results = {}
+    for sec_id, sec_text in mdx.items():
+        if sec_id not in gt_by_id:
+            continue
+        ranked = fn(sec_text, figma)
+        results[sec_id] = {
+            "ranked": ranked[:5],
+            "eval": evaluate_ranking(gt_by_id[sec_id], ranked),
+        }
+    return {"method": name, "results": results}
+
+
+def run_ai_metadata(name, mdx, gt_list, metadata_db):
+    gt_by_id = {g["id"]: g for g in gt_list}
+    frame_meta = metadata_db["figma_frames"]
+    mdx_meta = metadata_db["mdx_sections"]
+    results = {}
+    for sec_id in gt_by_id:
+        ranked = method_ai_metadata_matcher(sec_id, frame_meta, mdx_meta)
+        results[sec_id] = {
+            "ranked": ranked[:5],
+            "eval": evaluate_ranking(gt_by_id[sec_id], ranked),
+        }
+    return {"method": name, "results": results}
+
+
+def main():
+    print("[1/3] 데이터 로딩 중...")
+    gt = load_ground_truth()
+    figma = load_figma_texts()
+    mdx = load_mdx_sections()
+
+    meta_db_path = Path(__file__).parent / "metadata_db.yaml"
+    with open(meta_db_path, encoding="utf-8") as f:
+        metadata_db = yaml.safe_load(f)
+
+    print(f"  GT: {len(gt)} sections / Figma: {len(figma)} frames / MDX: {len(mdx)} sections")
+
+    print("[2/3] 8개 방법 실행 중...")
+    all_results = []
+    for name, fn in [
+        ("1. TF-IDF", method_tfidf),
+        ("2. BM25", method_bm25),
+        ("3. Char 2~3gram", method_char_ngram),
+        ("4. Kiwi+BM25", method_kiwi_bm25),
+        ("5. Structural", method_structural),
+        ("7. Weighted (BM25+CNG)", method_weighted),
+        ("8. HardFilter+BM25", method_hard_filter),
+    ]:
+        print(f"  실행: {name}")
+        all_results.append(run_method_standard(name, fn, mdx, figma, gt))
+
+    # 6번은 별도 함수
+    print(f"  실행: 6. AI-Metadata")
+    all_results.insert(5, run_ai_metadata("6. AI-Metadata", mdx, gt, metadata_db))
+
+    print("[3/3] 리포트 생성 중...")
+    report = generate_report(all_results, gt, mdx)
+    out_path = Path(__file__).parent / "RESULT.md"
+    out_path.write_text(report, encoding="utf-8")
+    print(f"\n완료: {out_path}")
+
+    # 콘솔에도 요약 출력
+    print("\n=== 방법별 요약 (null GT 제외) ===")
+    for r in all_results:
+        m = aggregate_metrics(r)
+        print(f"  {r['method']:25s}  Hit@1={m['hit@1']:.2f}  Hit@3={m['hit@3']:.2f}  MRR={m['mrr']:.3f}")
+
+
+def generate_report(all_results, gt, mdx_sections):
+    lines = []
+    lines.append("# MDX ↔ Figma 매칭 방법 비교 결과")
+    lines.append("")
+    lines.append(f"테스트 셋: MDX 01~03의 `##` 중목차 **{len(gt)}개 섹션**")
+    lines.append(f"Figma 프레임 **32개** 중 매칭")
+    lines.append(f"비교 방법 **{len(all_results)}개**")
+    lines.append("")
+
+    lines.append("## 1. Ground Truth")
+    lines.append("")
+    lines.append("| 섹션 ID | MDX 섹션 | Primary | Secondary | 확신도 |")
+    lines.append("|---------|---------|---------|-----------|--------|")
+    for g in gt:
+        prim = g["primary"] if g["primary"] else "null"
+        sec = ", ".join(str(s) for s in (g.get("secondary") or [])) or "-"
+        lines.append(f"| {g['id']} | {g['section_title']} | {prim} | {sec} | {g['confidence']} |")
+    lines.append("")
+
+    lines.append("## 2. 메인 매트릭스 — 각 방법의 Top-1")
+    lines.append("")
+    sec_ids = [g["id"] for g in gt]
+    header = "| 방법 | " + " | ".join(s.replace("MDX", "M") for s in sec_ids) + " |"
+    lines.append(header)
+    lines.append("|" + "---|" * (len(sec_ids) + 1))
+
+    # GT row
+    gt_row = ["**GT**"]
+    for g in gt:
+        p = g["primary"] if g["primary"] else "_null_"
+        gt_row.append(str(p))
+    lines.append("| " + " | ".join(gt_row) + " |")
+
+    # method rows
+    gt_by_id = {g["id"]: g for g in gt}
+    for r in all_results:
+        row = [r["method"]]
+        for sid in sec_ids:
+            res = r["results"].get(sid)
+            if not res or not res["ranked"]:
+                row.append("-")
+                continue
+            top_id, top_score = res["ranked"][0]
+            gt_entry = gt_by_id[sid]
+            gt_set = set()
+            if gt_entry["primary"]:
+                gt_set.add(str(gt_entry["primary"]))
+            gt_set |= {str(s) for s in (gt_entry.get("secondary") or [])}
+
+            mark = "✅" if str(top_id) == str(gt_entry["primary"]) else ("◯" if str(top_id) in gt_set else "✗")
+            if gt_entry["primary"] is None:
+                mark = "·"
+            short_id = str(top_id).replace("1171281", "")
+            row.append(f"{mark}{short_id} ({top_score:.2f})")
+        lines.append("| " + " | ".join(row) + " |")
+    lines.append("")
+    lines.append("✅ primary 일치 · ◯ secondary 일치 · ✗ 불일치 · · GT null")
+    lines.append("")
+
+    lines.append("## 3. Top-3 분석 (명확 매칭 5개 섹션만)")
+    lines.append("")
+    clear_sections = [g["id"] for g in gt if g["primary"] is not None]
+    for r in all_results:
+        lines.append(f"### {r['method']}")
+        lines.append("")
+        lines.append("| 섹션 | Top-1 | Top-2 | Top-3 |")
+        lines.append("|------|-------|-------|-------|")
+        for sid in clear_sections:
+            res = r["results"].get(sid)
+            if not res:
+                continue
+            ranked = res["ranked"][:3]
+            row = [sid]
+            for fid, sc in ranked:
+                short = str(fid).replace("1171281", "")
+                row.append(f"{short} ({sc:.2f})")
+            while len(row) < 4:
+                row.append("-")
+            lines.append("| " + " | ".join(row) + " |")
+        lines.append("")
+
+    lines.append("## 4. 지표 요약")
+    lines.append("")
+    lines.append("| 방법 | Hit@1 | Hit@3 | MRR | 명확5개 맞춤 |")
+    lines.append("|------|:---:|:---:|:---:|:---:|")
+    for r in all_results:
+        m = aggregate_metrics(r)
+        # 명확 5개 중 몇 개 primary 맞췄나
+        clear_hit = 0
+        for sid in clear_sections:
+            res = r["results"].get(sid)
+            if res and res["eval"]["hit@1"]:
+                clear_hit += 1
+        lines.append(
+            f"| {r['method']} | {m['hit@1']:.2f} | {m['hit@3']:.2f} | {m['mrr']:.3f} | {clear_hit}/5 |"
+        )
+    lines.append("")
+
+    lines.append("## 5. 권고")
+    lines.append("")
+    # best method by hit@1
+    best = max(all_results, key=lambda r: aggregate_metrics(r)["hit@1"])
+    lines.append(f"- **최고 Hit@1 방법**: {best['method']} (Hit@1={aggregate_metrics(best)['hit@1']:.2f})")
+    best_mrr = max(all_results, key=lambda r: aggregate_metrics(r)["mrr"])
+    lines.append(f"- **최고 MRR 방법**: {best_mrr['method']} (MRR={aggregate_metrics(best_mrr)['mrr']:.3f})")
+    lines.append("")
+    lines.append("(상세 분석은 메인 매트릭스와 Top-3 참고)")
+
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    main()