189 lines
7.4 KiB
Python
189 lines
7.4 KiB
Python
"""Phase 11 — 두 가지 신규 전략 비교
|
||
A. Asymmetric: MDX=제목만 × Figma=전체(전처리됨)
|
||
B. Ensemble: Track A(BM25 본문 × 전체) + Track B(Cross 제목 × 제목) 합산
|
||
"""
|
||
import sys
|
||
import json
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).parent))
|
||
|
||
from common import load_figma_texts
|
||
from extract_units import extract_units
|
||
from methods import (
|
||
method_tfidf, method_char_ngram, method_kiwi_bm25,
|
||
method_distinctive_kiwi,
|
||
method_sbert_chunk, method_e5_chunk, method_cross_encoder,
|
||
method_hybrid_bm25_cross,
|
||
)
|
||
from phase10 import extract_titles_only_mdx, extract_titles_only_figma, TARGET_UNITS
|
||
|
||
ROOT = Path(r"d:\ad-hoc\kei\design_agent")
|
||
PREVIEW_DIR = ROOT / "data" / "figma_previews"
|
||
|
||
|
||
def short_id(fid, frame_to_short):
|
||
return frame_to_short.get(str(fid), "?")
|
||
|
||
|
||
def normalize_scores(scored):
|
||
"""score list [(fid, score), ...] → 0-1 정규화"""
|
||
if not scored:
|
||
return {}
|
||
max_s = max(s for _, s in scored)
|
||
min_s = min(s for _, s in scored)
|
||
if max_s == min_s:
|
||
return {fid: 0.5 for fid, _ in scored}
|
||
return {fid: (s - min_s) / (max_s - min_s) for fid, s in scored}
|
||
|
||
|
||
def method_ensemble(mdx_full, mdx_title, figma_full, figma_titles, alpha=0.5):
|
||
"""Track A: BM25(MDX 전체, Figma 전체) + Track B: Cross(MDX 제목, Figma 제목) 합산
|
||
alpha: Track A 가중치 (1-alpha: Track B)"""
|
||
track_a = method_kiwi_bm25(mdx_full, figma_full)
|
||
track_b = method_cross_encoder(mdx_title, figma_titles)
|
||
norm_a = normalize_scores(track_a)
|
||
norm_b = normalize_scores(track_b)
|
||
all_fids = set(norm_a.keys()) | set(norm_b.keys())
|
||
combined = []
|
||
for fid in all_fids:
|
||
a = norm_a.get(fid, 0)
|
||
b = norm_b.get(fid, 0)
|
||
combined.append((fid, alpha * a + (1 - alpha) * b))
|
||
return sorted(combined, key=lambda x: -x[1])
|
||
|
||
|
||
def main():
|
||
units_full = extract_units() # MDX 본문+제목 3x 가중
|
||
figma_full = load_figma_texts() # Figma 전체, 전처리+타이틀 가중
|
||
figma_titles = extract_titles_only_figma() # Figma 제목만
|
||
|
||
# MDX 제목만 (Phase 10 포맷)
|
||
mdx_titles = {}
|
||
for uid, _, _, fname, mid, sub in TARGET_UNITS:
|
||
mdx_titles[uid] = extract_titles_only_mdx(fname, mid, sub)
|
||
|
||
with open(PREVIEW_DIR / "index.json", encoding="utf-8") as f:
|
||
idx_data = json.load(f)
|
||
frame_to_short = {info["frame_id"]: sid for sid, info in idx_data.items()}
|
||
|
||
# 전략: (이름, fn(unit_id) → ranked list)
|
||
def strategy_asymmetric(method_fn, name):
|
||
def run(uid):
|
||
return method_fn(mdx_titles[uid], figma_full)
|
||
return (f"{name}: MDX제목 × Figma전체", run)
|
||
|
||
def strategy_phase10(method_fn, name):
|
||
# MDX 제목 × Figma 제목 (Phase 10)
|
||
def run(uid):
|
||
return method_fn(mdx_titles[uid], figma_titles)
|
||
return (f"{name}: MDX제목 × Figma제목 (Phase 10)", run)
|
||
|
||
def strategy_phase9(method_fn, name):
|
||
def run(uid):
|
||
return method_fn(units_full[uid], figma_full)
|
||
return (f"{name}: MDX전체 × Figma전체 (Phase 9)", run)
|
||
|
||
strategies = []
|
||
# 1. Asymmetric (신규 A안)
|
||
strategies.append(strategy_asymmetric(method_kiwi_bm25, "BM25"))
|
||
strategies.append(strategy_asymmetric(method_cross_encoder, "Cross"))
|
||
strategies.append(strategy_asymmetric(method_sbert_chunk, "SBERT-청크"))
|
||
strategies.append(strategy_asymmetric(method_e5_chunk, "E5-청크"))
|
||
strategies.append(strategy_asymmetric(lambda m, f: method_hybrid_bm25_cross(m, f, top_k=5), "BM25→Cross"))
|
||
|
||
# 2. Ensemble (신규 B안) — alpha 3개로 샘플
|
||
for a in [0.3, 0.5, 0.7]:
|
||
name = f"Ensemble α={a} (A=BM25본문×전체 + B=Cross제목×제목)"
|
||
def make_run(alpha):
|
||
def run(uid):
|
||
return method_ensemble(units_full[uid], mdx_titles[uid],
|
||
figma_full, figma_titles, alpha=alpha)
|
||
return run
|
||
strategies.append((name, make_run(a)))
|
||
|
||
# 비교용 베이스라인
|
||
strategies.append(strategy_phase9(method_kiwi_bm25, "[기준] BM25"))
|
||
strategies.append(strategy_phase9(method_cross_encoder, "[기준] Cross"))
|
||
strategies.append(strategy_phase9(lambda m, f: method_hybrid_bm25_cross(m, f, top_k=5), "[기준] BM25→Cross"))
|
||
|
||
print("=" * 70)
|
||
print("Phase 11 — Asymmetric + Ensemble 실험")
|
||
print("=" * 70)
|
||
|
||
results_by_strategy = {}
|
||
for sname, run_fn in strategies:
|
||
top3_by = {}
|
||
results = []
|
||
for uid, _, correct_id, _, _, _ in TARGET_UNITS:
|
||
try:
|
||
ranked = run_fn(uid)
|
||
top3 = [short_id(fid, frame_to_short) for fid, _ in ranked[:3]]
|
||
except Exception as e:
|
||
top3 = ["ERR", "-", "-"]
|
||
print(f"ERR {sname} {uid}: {e}")
|
||
top3_by[uid] = top3
|
||
results.append(top3[0] == correct_id)
|
||
score = sum(results)
|
||
marks = ["✓" if r else "✗" for r in results]
|
||
results_by_strategy[sname] = (results, top3_by)
|
||
print(f" {sname[:55]:55s} {' '.join(marks)} = {score}/4")
|
||
|
||
# MD 리포트
|
||
png_rel = "../../data/figma_previews/"
|
||
lines = []
|
||
lines.append("# Phase 11 — Asymmetric & Ensemble 실험")
|
||
lines.append("")
|
||
lines.append("**신규 A 안**: MDX 제목만 × Figma 전체(전처리됨) — 비대칭 매칭")
|
||
lines.append("**신규 B 안**: BM25(MDX전체×Figma전체) + Cross(MDX제목×Figma제목) 점수 합산")
|
||
lines.append("")
|
||
|
||
lines.append("### 전략 × 4유닛 — 1순위 정답률")
|
||
lines.append("")
|
||
lines.append("| 전략 | MDX1 팝업(18) | MDX2 2.2(14) | MDX03-1(13) | MDX03-2(29) | 합계 |")
|
||
lines.append("|------|---------------|--------------|-------------|-------------|------|")
|
||
for sname, _ in strategies:
|
||
results, top3_by = results_by_strategy[sname]
|
||
marks = []
|
||
for idx, (uid, _, correct_id, _, _, _) in enumerate(TARGET_UNITS):
|
||
t3 = top3_by[uid]
|
||
mark = "✓" if results[idx] else f"✗({t3[0]})"
|
||
marks.append(mark)
|
||
score = sum(results)
|
||
lines.append(f"| {sname} | {marks[0]} | {marks[1]} | {marks[2]} | {marks[3]} | **{score}/4** |")
|
||
lines.append("")
|
||
|
||
# 유닛별 상세 (썸네일)
|
||
lines.append("---")
|
||
lines.append("")
|
||
lines.append("## 유닛별 전략 Top-3 상세")
|
||
lines.append("")
|
||
for uid, display, correct_id, _, _, _ in TARGET_UNITS:
|
||
lines.append(f"### {display} — 정답 Frame **{correct_id}**")
|
||
lines.append("")
|
||
lines.append("| 전략 | 1순위 | 2순위 | 3순위 |")
|
||
lines.append("|------|-------|-------|-------|")
|
||
for sname, _ in strategies:
|
||
_, top3_by = results_by_strategy[sname]
|
||
t3 = top3_by[uid]
|
||
row = [sname]
|
||
for sid in t3:
|
||
info = idx_data.get(sid, {})
|
||
png = info.get("png", "")
|
||
title = info.get("title_text", "").strip().replace("\n", " ") or ""
|
||
if len(title) > 15:
|
||
title = title[:15] + "…"
|
||
mark = "⭐ " if sid == correct_id else ""
|
||
cell = f"{mark}<br>**{sid}**<br>{title}"
|
||
row.append(cell)
|
||
lines.append("| " + " | ".join(row) + " |")
|
||
lines.append("")
|
||
|
||
out_path = Path(__file__).parent / "MATRIX_PHASE11.md"
|
||
out_path.write_text("\n".join(lines), encoding="utf-8")
|
||
print(f"\n완료: {out_path}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|