"""Phase 11 — 두 가지 신규 전략 비교 A. Asymmetric: MDX=제목만 × Figma=전체(전처리됨) B. Ensemble: Track A(BM25 본문 × 전체) + Track B(Cross 제목 × 제목) 합산 """ import sys import json from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from common import load_figma_texts from extract_units import extract_units from methods import ( method_tfidf, method_char_ngram, method_kiwi_bm25, method_distinctive_kiwi, method_sbert_chunk, method_e5_chunk, method_cross_encoder, method_hybrid_bm25_cross, ) from phase10 import extract_titles_only_mdx, extract_titles_only_figma, TARGET_UNITS ROOT = Path(r"d:\ad-hoc\kei\design_agent") PREVIEW_DIR = ROOT / "data" / "figma_previews" def short_id(fid, frame_to_short): return frame_to_short.get(str(fid), "?") def normalize_scores(scored): """score list [(fid, score), ...] → 0-1 정규화""" if not scored: return {} max_s = max(s for _, s in scored) min_s = min(s for _, s in scored) if max_s == min_s: return {fid: 0.5 for fid, _ in scored} return {fid: (s - min_s) / (max_s - min_s) for fid, s in scored} def method_ensemble(mdx_full, mdx_title, figma_full, figma_titles, alpha=0.5): """Track A: BM25(MDX 전체, Figma 전체) + Track B: Cross(MDX 제목, Figma 제목) 합산 alpha: Track A 가중치 (1-alpha: Track B)""" track_a = method_kiwi_bm25(mdx_full, figma_full) track_b = method_cross_encoder(mdx_title, figma_titles) norm_a = normalize_scores(track_a) norm_b = normalize_scores(track_b) all_fids = set(norm_a.keys()) | set(norm_b.keys()) combined = [] for fid in all_fids: a = norm_a.get(fid, 0) b = norm_b.get(fid, 0) combined.append((fid, alpha * a + (1 - alpha) * b)) return sorted(combined, key=lambda x: -x[1]) def main(): units_full = extract_units() # MDX 본문+제목 3x 가중 figma_full = load_figma_texts() # Figma 전체, 전처리+타이틀 가중 figma_titles = extract_titles_only_figma() # Figma 제목만 # MDX 제목만 (Phase 10 포맷) mdx_titles = {} for uid, _, _, fname, mid, sub in TARGET_UNITS: mdx_titles[uid] = extract_titles_only_mdx(fname, mid, sub) with open(PREVIEW_DIR / "index.json", encoding="utf-8") as f: idx_data = json.load(f) frame_to_short = {info["frame_id"]: sid for sid, info in idx_data.items()} # 전략: (이름, fn(unit_id) → ranked list) def strategy_asymmetric(method_fn, name): def run(uid): return method_fn(mdx_titles[uid], figma_full) return (f"{name}: MDX제목 × Figma전체", run) def strategy_phase10(method_fn, name): # MDX 제목 × Figma 제목 (Phase 10) def run(uid): return method_fn(mdx_titles[uid], figma_titles) return (f"{name}: MDX제목 × Figma제목 (Phase 10)", run) def strategy_phase9(method_fn, name): def run(uid): return method_fn(units_full[uid], figma_full) return (f"{name}: MDX전체 × Figma전체 (Phase 9)", run) strategies = [] # 1. Asymmetric (신규 A안) strategies.append(strategy_asymmetric(method_kiwi_bm25, "BM25")) strategies.append(strategy_asymmetric(method_cross_encoder, "Cross")) strategies.append(strategy_asymmetric(method_sbert_chunk, "SBERT-청크")) strategies.append(strategy_asymmetric(method_e5_chunk, "E5-청크")) strategies.append(strategy_asymmetric(lambda m, f: method_hybrid_bm25_cross(m, f, top_k=5), "BM25→Cross")) # 2. Ensemble (신규 B안) — alpha 3개로 샘플 for a in [0.3, 0.5, 0.7]: name = f"Ensemble α={a} (A=BM25본문×전체 + B=Cross제목×제목)" def make_run(alpha): def run(uid): return method_ensemble(units_full[uid], mdx_titles[uid], figma_full, figma_titles, alpha=alpha) return run strategies.append((name, make_run(a))) # 비교용 베이스라인 strategies.append(strategy_phase9(method_kiwi_bm25, "[기준] BM25")) strategies.append(strategy_phase9(method_cross_encoder, "[기준] Cross")) strategies.append(strategy_phase9(lambda m, f: method_hybrid_bm25_cross(m, f, top_k=5), "[기준] BM25→Cross")) print("=" * 70) print("Phase 11 — Asymmetric + Ensemble 실험") print("=" * 70) results_by_strategy = {} for sname, run_fn in strategies: top3_by = {} results = [] for uid, _, correct_id, _, _, _ in TARGET_UNITS: try: ranked = run_fn(uid) top3 = [short_id(fid, frame_to_short) for fid, _ in ranked[:3]] except Exception as e: top3 = ["ERR", "-", "-"] print(f"ERR {sname} {uid}: {e}") top3_by[uid] = top3 results.append(top3[0] == correct_id) score = sum(results) marks = ["✓" if r else "✗" for r in results] results_by_strategy[sname] = (results, top3_by) print(f" {sname[:55]:55s} {' '.join(marks)} = {score}/4") # MD 리포트 png_rel = "../../data/figma_previews/" lines = [] lines.append("# Phase 11 — Asymmetric & Ensemble 실험") lines.append("") lines.append("**신규 A 안**: MDX 제목만 × Figma 전체(전처리됨) — 비대칭 매칭") lines.append("**신규 B 안**: BM25(MDX전체×Figma전체) + Cross(MDX제목×Figma제목) 점수 합산") lines.append("") lines.append("### 전략 × 4유닛 — 1순위 정답률") lines.append("") lines.append("| 전략 | MDX1 팝업(18) | MDX2 2.2(14) | MDX03-1(13) | MDX03-2(29) | 합계 |") lines.append("|------|---------------|--------------|-------------|-------------|------|") for sname, _ in strategies: results, top3_by = results_by_strategy[sname] marks = [] for idx, (uid, _, correct_id, _, _, _) in enumerate(TARGET_UNITS): t3 = top3_by[uid] mark = "✓" if results[idx] else f"✗({t3[0]})" marks.append(mark) score = sum(results) lines.append(f"| {sname} | {marks[0]} | {marks[1]} | {marks[2]} | {marks[3]} | **{score}/4** |") lines.append("") # 유닛별 상세 (썸네일) lines.append("---") lines.append("") lines.append("## 유닛별 전략 Top-3 상세") lines.append("") for uid, display, correct_id, _, _, _ in TARGET_UNITS: lines.append(f"### {display} — 정답 Frame **{correct_id}**") lines.append("") lines.append("| 전략 | 1순위 | 2순위 | 3순위 |") lines.append("|------|-------|-------|-------|") for sname, _ in strategies: _, top3_by = results_by_strategy[sname] t3 = top3_by[uid] row = [sname] for sid in t3: info = idx_data.get(sid, {}) png = info.get("png", "") title = info.get("title_text", "").strip().replace("\n", " ") or "" if len(title) > 15: title = title[:15] + "…" mark = "⭐ " if sid == correct_id else "" cell = f"{mark}![{sid}]({png_rel}{png})
**{sid}**
{title}" row.append(cell) lines.append("| " + " | ".join(row) + " |") lines.append("") out_path = Path(__file__).parent / "MATRIX_PHASE11.md" out_path.write_text("\n".join(lines), encoding="utf-8") print(f"\n완료: {out_path}") if __name__ == "__main__": main()