"""8개 매칭 방법 실행 + 비교 매트릭스 리포트 생성""" import sys import os import yaml from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from common import ( load_ground_truth, load_figma_texts, load_mdx_sections, evaluate_ranking, aggregate_metrics ) from methods import ( method_tfidf, method_bm25, method_char_ngram, method_kiwi_bm25, method_structural, method_ai_metadata_matcher, method_weighted, method_hard_filter, ) def run_method_standard(name, fn, mdx, figma, gt_list): gt_by_id = {g["id"]: g for g in gt_list} results = {} for sec_id, sec_text in mdx.items(): if sec_id not in gt_by_id: continue ranked = fn(sec_text, figma) results[sec_id] = { "ranked": ranked[:5], "eval": evaluate_ranking(gt_by_id[sec_id], ranked), } return {"method": name, "results": results} def run_ai_metadata(name, mdx, gt_list, metadata_db): gt_by_id = {g["id"]: g for g in gt_list} frame_meta = metadata_db["figma_frames"] mdx_meta = metadata_db["mdx_sections"] results = {} for sec_id in gt_by_id: ranked = method_ai_metadata_matcher(sec_id, frame_meta, mdx_meta) results[sec_id] = { "ranked": ranked[:5], "eval": evaluate_ranking(gt_by_id[sec_id], ranked), } return {"method": name, "results": results} def main(): print("[1/3] 데이터 로딩 중...") gt = load_ground_truth() figma = load_figma_texts() mdx = load_mdx_sections() meta_db_path = Path(__file__).parent / "metadata_db.yaml" with open(meta_db_path, encoding="utf-8") as f: metadata_db = yaml.safe_load(f) print(f" GT: {len(gt)} sections / Figma: {len(figma)} frames / MDX: {len(mdx)} sections") print("[2/3] 8개 방법 실행 중...") all_results = [] for name, fn in [ ("1. TF-IDF", method_tfidf), ("2. BM25", method_bm25), ("3. Char 2~3gram", method_char_ngram), ("4. Kiwi+BM25", method_kiwi_bm25), ("5. Structural", method_structural), ("7. Weighted (BM25+CNG)", method_weighted), ("8. HardFilter+BM25", method_hard_filter), ]: print(f" 실행: {name}") all_results.append(run_method_standard(name, fn, mdx, figma, gt)) # 6번은 별도 함수 print(f" 실행: 6. AI-Metadata") all_results.insert(5, run_ai_metadata("6. AI-Metadata", mdx, gt, metadata_db)) print("[3/3] 리포트 생성 중...") report = generate_report(all_results, gt, mdx) out_path = Path(__file__).parent / "RESULT.md" out_path.write_text(report, encoding="utf-8") print(f"\n완료: {out_path}") # 콘솔에도 요약 출력 print("\n=== 방법별 요약 (null GT 제외) ===") for r in all_results: m = aggregate_metrics(r) print(f" {r['method']:25s} Hit@1={m['hit@1']:.2f} Hit@3={m['hit@3']:.2f} MRR={m['mrr']:.3f}") def generate_report(all_results, gt, mdx_sections): lines = [] lines.append("# MDX ↔ Figma 매칭 방법 비교 결과") lines.append("") lines.append(f"테스트 셋: MDX 01~03의 `##` 중목차 **{len(gt)}개 섹션**") lines.append(f"Figma 프레임 **32개** 중 매칭") lines.append(f"비교 방법 **{len(all_results)}개**") lines.append("") lines.append("## 1. Ground Truth") lines.append("") lines.append("| 섹션 ID | MDX 섹션 | Primary | Secondary | 확신도 |") lines.append("|---------|---------|---------|-----------|--------|") for g in gt: prim = g["primary"] if g["primary"] else "null" sec = ", ".join(str(s) for s in (g.get("secondary") or [])) or "-" lines.append(f"| {g['id']} | {g['section_title']} | {prim} | {sec} | {g['confidence']} |") lines.append("") lines.append("## 2. 메인 매트릭스 — 각 방법의 Top-1") lines.append("") sec_ids = [g["id"] for g in gt] header = "| 방법 | " + " | ".join(s.replace("MDX", "M") for s in sec_ids) + " |" lines.append(header) lines.append("|" + "---|" * (len(sec_ids) + 1)) # GT row gt_row = ["**GT**"] for g in gt: p = g["primary"] if g["primary"] else "_null_" gt_row.append(str(p)) lines.append("| " + " | ".join(gt_row) + " |") # method rows gt_by_id = {g["id"]: g for g in gt} for r in all_results: row = [r["method"]] for sid in sec_ids: res = r["results"].get(sid) if not res or not res["ranked"]: row.append("-") continue top_id, top_score = res["ranked"][0] gt_entry = gt_by_id[sid] gt_set = set() if gt_entry["primary"]: gt_set.add(str(gt_entry["primary"])) gt_set |= {str(s) for s in (gt_entry.get("secondary") or [])} mark = "✅" if str(top_id) == str(gt_entry["primary"]) else ("◯" if str(top_id) in gt_set else "✗") if gt_entry["primary"] is None: mark = "·" short_id = str(top_id).replace("1171281", "") row.append(f"{mark}{short_id} ({top_score:.2f})") lines.append("| " + " | ".join(row) + " |") lines.append("") lines.append("✅ primary 일치 · ◯ secondary 일치 · ✗ 불일치 · · GT null") lines.append("") lines.append("## 3. Top-3 분석 (명확 매칭 5개 섹션만)") lines.append("") clear_sections = [g["id"] for g in gt if g["primary"] is not None] for r in all_results: lines.append(f"### {r['method']}") lines.append("") lines.append("| 섹션 | Top-1 | Top-2 | Top-3 |") lines.append("|------|-------|-------|-------|") for sid in clear_sections: res = r["results"].get(sid) if not res: continue ranked = res["ranked"][:3] row = [sid] for fid, sc in ranked: short = str(fid).replace("1171281", "") row.append(f"{short} ({sc:.2f})") while len(row) < 4: row.append("-") lines.append("| " + " | ".join(row) + " |") lines.append("") lines.append("## 4. 지표 요약") lines.append("") lines.append("| 방법 | Hit@1 | Hit@3 | MRR | 명확5개 맞춤 |") lines.append("|------|:---:|:---:|:---:|:---:|") for r in all_results: m = aggregate_metrics(r) # 명확 5개 중 몇 개 primary 맞췄나 clear_hit = 0 for sid in clear_sections: res = r["results"].get(sid) if res and res["eval"]["hit@1"]: clear_hit += 1 lines.append( f"| {r['method']} | {m['hit@1']:.2f} | {m['hit@3']:.2f} | {m['mrr']:.3f} | {clear_hit}/5 |" ) lines.append("") lines.append("## 5. 권고") lines.append("") # best method by hit@1 best = max(all_results, key=lambda r: aggregate_metrics(r)["hit@1"]) lines.append(f"- **최고 Hit@1 방법**: {best['method']} (Hit@1={aggregate_metrics(best)['hit@1']:.2f})") best_mrr = max(all_results, key=lambda r: aggregate_metrics(r)["mrr"]) lines.append(f"- **최고 MRR 방법**: {best_mrr['method']} (MRR={aggregate_metrics(best_mrr)['mrr']:.3f})") lines.append("") lines.append("(상세 분석은 메인 매트릭스와 Top-3 참고)") return "\n".join(lines) if __name__ == "__main__": main()