"""MDX 섹션 중심의 결과 재편성 리포트 (figma_previews index 기반)""" import sys import json import yaml from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from common import load_ground_truth, load_figma_texts, load_mdx_sections from methods import ( method_tfidf, method_bm25, method_char_ngram, method_kiwi_bm25, method_structural, method_ai_metadata_matcher, method_weighted, method_hard_filter, ) ROOT = Path(r"d:\ad-hoc\kei\design_agent") PREVIEW_DIR = ROOT / "data" / "figma_previews" def load_frame_index(): """index.json: short_id(01~32) ↔ frame_id ↔ title ↔ png""" with open(PREVIEW_DIR / "index.json", encoding="utf-8") as f: data = json.load(f) # frame_id → short_id 역매핑 frame_to_short = {} short_to_info = {} for short_id, info in data.items(): frame_to_short[info["frame_id"]] = short_id short_to_info[short_id] = info return frame_to_short, short_to_info def fmt_frame(fid, frame_to_short, short_to_info, score=None, mark=""): """frame_id → 'N번 제목' + 선택적 score""" fid_s = str(fid) short = frame_to_short.get(fid_s, "?") title = short_to_info.get(short, {}).get("title_text", "").strip() # 제목이 너무 길면 자름 if len(title) > 25: title = title[:25] + "…" if not title: title = f"#{fid_s[-4:]}" # 제목 없으면 번호 뒤 4자리 s = f"{mark}**{short}** {title}" if score is not None: s += f" ({score:.2f})" return s def main(): gt_list = load_ground_truth() figma = load_figma_texts() mdx = load_mdx_sections() frame_to_short, short_to_info = load_frame_index() with open(Path(__file__).parent / "metadata_db.yaml", encoding="utf-8") as f: metadata_db = yaml.safe_load(f) methods_def = [ ("TF-IDF", method_tfidf), ("BM25", method_bm25), ("Char-ngram", method_char_ngram), ("Kiwi+BM25", method_kiwi_bm25), ("Structural", method_structural), ("AI-Metadata", None), ("Weighted", method_weighted), ("HardFilter", method_hard_filter), ] all_results = {} for sec_id, sec_text in mdx.items(): all_results[sec_id] = {} for name, fn in methods_def: if name == "AI-Metadata": ranked = method_ai_metadata_matcher( sec_id, metadata_db["figma_frames"], metadata_db["mdx_sections"] ) else: ranked = fn(sec_text, figma) all_results[sec_id][name] = ranked[:3] # 리포트 생성 lines = [] lines.append("# MDX 섹션별 매칭 결과 (8개 방법 비교)") lines.append("") lines.append("각 MDX 섹션 ↔ Figma 프레임 매칭. 프레임 번호는 `data/figma_previews/index.json` 기준 01~32.") lines.append("") lines.append("**표기**: ✅ GT primary 일치 · ◯ GT secondary 일치 · ✗ 불일치 · · GT=null") lines.append("") # 프레임 번호 전체 매핑 표 (참고용) lines.append("## 프레임 번호 매핑 (참고)") lines.append("") lines.append("
01~32 번호 ↔ 프레임 ID ↔ 제목 (펼치기)") lines.append("") lines.append("| # | Frame ID | 제목 | 미리보기 |") lines.append("|---|----------|------|---------|") for short_id in sorted(short_to_info.keys()): info = short_to_info[short_id] title = info.get("title_text", "").strip().replace("\n", " ") or "_(제목없음)_" if len(title) > 40: title = title[:40] + "…" png_rel = f"../../data/figma_previews/{info['png']}" lines.append(f"| **{short_id}** | {info['frame_id']} | {title} | ![{short_id}]({png_rel}) |") lines.append("") lines.append("
") lines.append("") # 각 MDX 섹션별 for gt in gt_list: sid = gt["id"] lines.append(f"---") lines.append("") lines.append(f"## {sid} — {gt['section_title']}") lines.append("") if sid in mdx: preview = mdx[sid][:250].replace("\n", " ").strip() lines.append(f"> **MDX 내용**: {preview}...") lines.append("") # Ground Truth gt_lines = [] if gt["primary"]: prim_str = fmt_frame(gt["primary"], frame_to_short, short_to_info) gt_lines.append(f"- **Primary**: {prim_str}") else: gt_lines.append(f"- **Primary**: `null` (매칭 없음)") if gt.get("secondary"): secs = [fmt_frame(s, frame_to_short, short_to_info) for s in gt["secondary"]] gt_lines.append(f"- **Secondary**: {', '.join(secs)}") gt_lines.append(f"- **확신도**: {gt['confidence']}") gt_lines.append(f"- **주석**: {gt.get('note', '-')}") lines.append("**Ground Truth**") lines.append("") for gl in gt_lines: lines.append(gl) lines.append("") # GT 프레임 미리보기 if gt["primary"]: gt_short = frame_to_short.get(str(gt["primary"])) if gt_short: info = short_to_info[gt_short] png_rel = f"../../data/figma_previews/{info['png']}" lines.append(f"![GT {gt_short}]({png_rel})") lines.append("") # 8개 방법 Top-3 lines.append("### 방법별 Top-3") lines.append("") lines.append("| 방법 | Top-1 | Top-2 | Top-3 |") lines.append("|------|-------|-------|-------|") gt_set = set() if gt["primary"]: gt_set.add(str(gt["primary"])) gt_set |= {str(s) for s in (gt.get("secondary") or [])} for name, _ in methods_def: top3 = all_results[sid][name] row = [name] for fid, score in top3: fid_s = str(fid) if gt["primary"] is None: mark = "· " elif fid_s == str(gt["primary"]): mark = "✅ " elif fid_s in gt_set: mark = "◯ " else: mark = "✗ " row.append(fmt_frame(fid_s, frame_to_short, short_to_info, score=score, mark=mark)) while len(row) < 4: row.append("-") lines.append("| " + " | ".join(row) + " |") lines.append("") # Top-1 집계 top1_match = {} for name, _ in methods_def: top_fid = str(all_results[sid][name][0][0]) if all_results[sid][name] else "-" top1_match.setdefault(top_fid, []).append(name) lines.append("**Top-1 집계**") lines.append("") for fid, names in sorted(top1_match.items(), key=lambda x: -len(x[1])): if gt["primary"] is None: emoji = "·" elif fid == str(gt["primary"]): emoji = "✅" elif fid in gt_set: emoji = "◯" else: emoji = "✗" fname = fmt_frame(fid, frame_to_short, short_to_info) lines.append(f"- {emoji} {fname} ← {', '.join(names)} ({len(names)}개 방법)") lines.append("") # 마지막: 방법별 요약 lines.append("---") lines.append("") lines.append("## 방법별 정답률 요약") lines.append("") # 명확 GT(primary != null) 기준 Hit@1 clear_sections = [g for g in gt_list if g["primary"] is not None] lines.append("| 방법 | 정답률 | 맞춘 섹션 |") lines.append("|------|:---:|----------|") for name, _ in methods_def: hits = [] for g in clear_sections: top1 = all_results[g["id"]][name][0][0] if all_results[g["id"]][name] else None if str(top1) == str(g["primary"]): hits.append(g["id"]) lines.append(f"| {name} | {len(hits)}/5 | {', '.join(hits) if hits else '-'} |") lines.append("") out_path = Path(__file__).parent / "RESULT_BY_SECTION.md" out_path.write_text("\n".join(lines), encoding="utf-8") print(f"완료: {out_path}") if __name__ == "__main__": main()