C.E.L_Slide_test2/tests/matching/run_all.py

"""8개 매칭 방법 실행 + 비교 매트릭스 리포트 생성"""
import sys
import os
import yaml
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))

from common import (
    load_ground_truth, load_figma_texts, load_mdx_sections,
    evaluate_ranking, aggregate_metrics
)
from methods import (
    method_tfidf, method_bm25, method_char_ngram,
    method_kiwi_bm25, method_structural,
    method_ai_metadata_matcher, method_weighted, method_hard_filter,
)


def run_method_standard(name, fn, mdx, figma, gt_list):
    gt_by_id = {g["id"]: g for g in gt_list}
    results = {}
    for sec_id, sec_text in mdx.items():
        if sec_id not in gt_by_id:
            continue
        ranked = fn(sec_text, figma)
        results[sec_id] = {
            "ranked": ranked[:5],
            "eval": evaluate_ranking(gt_by_id[sec_id], ranked),
        }
    return {"method": name, "results": results}


def run_ai_metadata(name, mdx, gt_list, metadata_db):
    gt_by_id = {g["id"]: g for g in gt_list}
    frame_meta = metadata_db["figma_frames"]
    mdx_meta = metadata_db["mdx_sections"]
    results = {}
    for sec_id in gt_by_id:
        ranked = method_ai_metadata_matcher(sec_id, frame_meta, mdx_meta)
        results[sec_id] = {
            "ranked": ranked[:5],
            "eval": evaluate_ranking(gt_by_id[sec_id], ranked),
        }
    return {"method": name, "results": results}


def main():
    print("[1/3] 데이터 로딩 중...")
    gt = load_ground_truth()
    figma = load_figma_texts()
    mdx = load_mdx_sections()

    meta_db_path = Path(__file__).parent / "metadata_db.yaml"
    with open(meta_db_path, encoding="utf-8") as f:
        metadata_db = yaml.safe_load(f)

    print(f"  GT: {len(gt)} sections / Figma: {len(figma)} frames / MDX: {len(mdx)} sections")

    print("[2/3] 8개 방법 실행 중...")
    all_results = []
    for name, fn in [
        ("1. TF-IDF", method_tfidf),
        ("2. BM25", method_bm25),
        ("3. Char 2~3gram", method_char_ngram),
        ("4. Kiwi+BM25", method_kiwi_bm25),
        ("5. Structural", method_structural),
        ("7. Weighted (BM25+CNG)", method_weighted),
        ("8. HardFilter+BM25", method_hard_filter),
    ]:
        print(f"  실행: {name}")
        all_results.append(run_method_standard(name, fn, mdx, figma, gt))

    # 6번은 별도 함수
    print(f"  실행: 6. AI-Metadata")
    all_results.insert(5, run_ai_metadata("6. AI-Metadata", mdx, gt, metadata_db))

    print("[3/3] 리포트 생성 중...")
    report = generate_report(all_results, gt, mdx)
    out_path = Path(__file__).parent / "RESULT.md"
    out_path.write_text(report, encoding="utf-8")
    print(f"\n완료: {out_path}")

    # 콘솔에도 요약 출력
    print("\n=== 방법별 요약 (null GT 제외) ===")
    for r in all_results:
        m = aggregate_metrics(r)
        print(f"  {r['method']:25s}  Hit@1={m['hit@1']:.2f}  Hit@3={m['hit@3']:.2f}  MRR={m['mrr']:.3f}")


def generate_report(all_results, gt, mdx_sections):
    lines = []
    lines.append("# MDX ↔ Figma 매칭 방법 비교 결과")
    lines.append("")
    lines.append(f"테스트 셋: MDX 01~03의 `##` 중목차 **{len(gt)}개 섹션**")
    lines.append(f"Figma 프레임 **32개** 중 매칭")
    lines.append(f"비교 방법 **{len(all_results)}개**")
    lines.append("")

    lines.append("## 1. Ground Truth")
    lines.append("")
    lines.append("| 섹션 ID | MDX 섹션 | Primary | Secondary | 확신도 |")
    lines.append("|---------|---------|---------|-----------|--------|")
    for g in gt:
        prim = g["primary"] if g["primary"] else "null"
        sec = ", ".join(str(s) for s in (g.get("secondary") or [])) or "-"
        lines.append(f"| {g['id']} | {g['section_title']} | {prim} | {sec} | {g['confidence']} |")
    lines.append("")

    lines.append("## 2. 메인 매트릭스 — 각 방법의 Top-1")
    lines.append("")
    sec_ids = [g["id"] for g in gt]
    header = "| 방법 | " + " | ".join(s.replace("MDX", "M") for s in sec_ids) + " |"
    lines.append(header)
    lines.append("|" + "---|" * (len(sec_ids) + 1))

    # GT row
    gt_row = ["**GT**"]
    for g in gt:
        p = g["primary"] if g["primary"] else "_null_"
        gt_row.append(str(p))
    lines.append("| " + " | ".join(gt_row) + " |")

    # method rows
    gt_by_id = {g["id"]: g for g in gt}
    for r in all_results:
        row = [r["method"]]
        for sid in sec_ids:
            res = r["results"].get(sid)
            if not res or not res["ranked"]:
                row.append("-")
                continue
            top_id, top_score = res["ranked"][0]
            gt_entry = gt_by_id[sid]
            gt_set = set()
            if gt_entry["primary"]:
                gt_set.add(str(gt_entry["primary"]))
            gt_set |= {str(s) for s in (gt_entry.get("secondary") or [])}

            mark = "✅" if str(top_id) == str(gt_entry["primary"]) else ("◯" if str(top_id) in gt_set else "✗")
            if gt_entry["primary"] is None:
                mark = "·"
            short_id = str(top_id).replace("1171281", "")
            row.append(f"{mark}{short_id} ({top_score:.2f})")
        lines.append("| " + " | ".join(row) + " |")
    lines.append("")
    lines.append("✅ primary 일치 · ◯ secondary 일치 · ✗ 불일치 · · GT null")
    lines.append("")

    lines.append("## 3. Top-3 분석 (명확 매칭 5개 섹션만)")
    lines.append("")
    clear_sections = [g["id"] for g in gt if g["primary"] is not None]
    for r in all_results:
        lines.append(f"### {r['method']}")
        lines.append("")
        lines.append("| 섹션 | Top-1 | Top-2 | Top-3 |")
        lines.append("|------|-------|-------|-------|")
        for sid in clear_sections:
            res = r["results"].get(sid)
            if not res:
                continue
            ranked = res["ranked"][:3]
            row = [sid]
            for fid, sc in ranked:
                short = str(fid).replace("1171281", "")
                row.append(f"{short} ({sc:.2f})")
            while len(row) < 4:
                row.append("-")
            lines.append("| " + " | ".join(row) + " |")
        lines.append("")

    lines.append("## 4. 지표 요약")
    lines.append("")
    lines.append("| 방법 | Hit@1 | Hit@3 | MRR | 명확5개 맞춤 |")
    lines.append("|------|:---:|:---:|:---:|:---:|")
    for r in all_results:
        m = aggregate_metrics(r)
        # 명확 5개 중 몇 개 primary 맞췄나
        clear_hit = 0
        for sid in clear_sections:
            res = r["results"].get(sid)
            if res and res["eval"]["hit@1"]:
                clear_hit += 1
        lines.append(
            f"| {r['method']} | {m['hit@1']:.2f} | {m['hit@3']:.2f} | {m['mrr']:.3f} | {clear_hit}/5 |"
        )
    lines.append("")

    lines.append("## 5. 권고")
    lines.append("")
    # best method by hit@1
    best = max(all_results, key=lambda r: aggregate_metrics(r)["hit@1"])
    lines.append(f"- **최고 Hit@1 방법**: {best['method']} (Hit@1={aggregate_metrics(best)['hit@1']:.2f})")
    best_mrr = max(all_results, key=lambda r: aggregate_metrics(r)["mrr"])
    lines.append(f"- **최고 MRR 방법**: {best_mrr['method']} (MRR={aggregate_metrics(best_mrr)['mrr']:.3f})")
    lines.append("")
    lines.append("(상세 분석은 메인 매트릭스와 Top-3 참고)")

    return "\n".join(lines)


if __name__ == "__main__":
    main()