196 lines
8.7 KiB
Python
196 lines
8.7 KiB
Python
"""Pipeline Step 18 — V4 slot 축 ablation (실증).
|
||
|
||
질문: V4 의 5축 중 slot 축 (W_SLOT=0.15) 이 frame 매칭에 실제로 기여하는가?
|
||
|
||
방법:
|
||
1. 기존 v4_full32_result (W_SLOT=0.15 유지) 를 baseline 으로
|
||
2. W_SLOT=0 으로 두고 나머지 4축 (anchor/cardinality/relation/content) 가중치
|
||
재정규화 — sum=1.0 유지 → ablated 결과 산출
|
||
3. 32 frame × 7 MDX 섹션 confidence 재계산
|
||
4. 비교:
|
||
- TARGET 4개의 정답 frame 라벨/순위 변화
|
||
- Top-3 매칭 변화
|
||
- 라벨 분포 변화 (use_as_is/light_edit/restructure/reject 카운트)
|
||
5. 결론: slot 축이 의미 있나?
|
||
|
||
출력: V4_SLOT_ABLATION.md
|
||
"""
|
||
import sys
|
||
from pathlib import Path
|
||
import yaml
|
||
|
||
HERE = Path(__file__).parent
|
||
sys.path.insert(0, str(HERE))
|
||
|
||
from template_fit import route
|
||
|
||
OUT_PATH = HERE / 'V4_SLOT_ABLATION.md'
|
||
|
||
# Baseline (현재): 0.25 + 0.20 + 0.20 + 0.15 + 0.20 = 1.00
|
||
W_BASELINE = {'anchor': 0.25, 'cardinality': 0.20, 'relation': 0.20, 'slot': 0.15, 'content': 0.20}
|
||
|
||
# Ablated: slot=0, 나머지 4축 비례 재정규화 → sum=1.00
|
||
# 4축 합 = 0.85, 각 축 / 0.85
|
||
W_ABLATED = {
|
||
'anchor': 0.25 / 0.85, # 0.2941
|
||
'cardinality': 0.20 / 0.85, # 0.2353
|
||
'relation': 0.20 / 0.85, # 0.2353
|
||
'slot': 0.0,
|
||
'content': 0.20 / 0.85, # 0.2353
|
||
}
|
||
|
||
|
||
def recompute_confidence(axes, weights, penalty):
|
||
base = sum(weights[k] * axes.get(k, 0) for k in weights)
|
||
return max(0.0, base - penalty)
|
||
|
||
|
||
def main():
|
||
v4 = yaml.safe_load((HERE / 'v4_full32_result.yaml').read_text(encoding='utf-8'))
|
||
answer_map = v4['meta']['answer_map']
|
||
holdout = set(v4['meta']['holdout_sections'])
|
||
|
||
md = []
|
||
md.append('# V4 slot 축 ablation — 실증\n')
|
||
md.append('## 질문\n')
|
||
md.append('V4 confidence 계산식의 slot 축 (W_SLOT=0.15) 이 frame 매칭에 실제로 기여하는가?\n')
|
||
md.append('## 방법\n')
|
||
md.append('- **Baseline**: 5축 가중치 그대로 — 0.25 anchor + 0.20 card + 0.20 rel + 0.15 slot + 0.20 content\n')
|
||
md.append('- **Ablated**: W_SLOT=0, 나머지 4축 비례 재정규화 (sum=1.0)\n')
|
||
md.append(' - anchor: 0.294, cardinality: 0.235, relation: 0.235, content: 0.235\n')
|
||
md.append('- 32 frame × 7 MDX 섹션 confidence 재계산 → label 재할당 → 비교\n\n')
|
||
|
||
section_results = {}
|
||
for sid, sec in v4['mdx_sections'].items():
|
||
is_target = sid not in holdout
|
||
ans_fn = answer_map.get(sid)
|
||
|
||
# Baseline 결과 (이미 v4_full32 에 있음)
|
||
baseline = sorted(sec['judgments_full32'], key=lambda x: -x['confidence'])
|
||
|
||
# Ablated 결과 재계산
|
||
ablated = []
|
||
for j in sec['judgments_full32']:
|
||
new_conf = recompute_confidence(j['axes'], W_ABLATED, j['penalty'])
|
||
new_label = route(new_conf)
|
||
ablated.append({
|
||
'frame_number': j['frame_number'],
|
||
'frame_id': j['frame_id'],
|
||
'baseline_conf': j['confidence'],
|
||
'baseline_label': j['label'],
|
||
'ablated_conf': round(new_conf, 4),
|
||
'ablated_label': new_label,
|
||
})
|
||
ablated.sort(key=lambda x: -x['ablated_conf'])
|
||
|
||
# 정답 frame 변화
|
||
ans_baseline = next((x for x in baseline if x['frame_number'] == ans_fn), None) if ans_fn else None
|
||
ans_ablated = next((x for x in ablated if x['frame_number'] == ans_fn), None) if ans_fn else None
|
||
|
||
# 순위 비교
|
||
baseline_top3 = [(x['frame_number'], x['confidence'], x['label']) for x in baseline[:3]]
|
||
ablated_top3 = [(x['frame_number'], x['ablated_conf'], x['ablated_label']) for x in ablated[:3]]
|
||
|
||
# 라벨 분포 비교
|
||
from collections import Counter
|
||
base_lbl = Counter(x['label'] for x in baseline)
|
||
abl_lbl = Counter(x['ablated_label'] for x in ablated)
|
||
|
||
section_results[sid] = {
|
||
'is_target': is_target,
|
||
'ans_fn': ans_fn,
|
||
'baseline_top3': baseline_top3,
|
||
'ablated_top3': ablated_top3,
|
||
'ans_baseline': ans_baseline,
|
||
'ans_ablated': ans_ablated,
|
||
'base_lbl': base_lbl,
|
||
'abl_lbl': abl_lbl,
|
||
}
|
||
|
||
# ─── TARGET 정답 매칭 변화 ───
|
||
md.append('## 결과 1 — TARGET 4 섹션의 정답 frame 라벨/순위 변화\n')
|
||
md.append('| MDX | 정답 Frame | Baseline conf / 라벨 / 순위 | Ablated conf / 라벨 / 순위 | 변화 |\n')
|
||
md.append('|---|---|---|---|---|\n')
|
||
target_sids = [sid for sid in section_results if section_results[sid]['is_target']]
|
||
correct_baseline = 0
|
||
correct_ablated = 0
|
||
for sid in target_sids:
|
||
r = section_results[sid]
|
||
ans_fn = r['ans_fn']
|
||
ab = r['ans_baseline']
|
||
aa = r['ans_ablated']
|
||
base_rank = next((i+1 for i, x in enumerate(sorted(v4['mdx_sections'][sid]['judgments_full32'], key=lambda x: -x['confidence'])) if x['frame_number'] == ans_fn), '?')
|
||
# ablated rank
|
||
ablated_sorted = sorted(
|
||
[{'fn': j['frame_number'], 'c': recompute_confidence(j['axes'], W_ABLATED, j['penalty'])}
|
||
for j in v4['mdx_sections'][sid]['judgments_full32']],
|
||
key=lambda x: -x['c']
|
||
)
|
||
abl_rank = next((i+1 for i, x in enumerate(ablated_sorted) if x['fn'] == ans_fn), '?')
|
||
|
||
# 정답률 — baseline / ablated 의 Top-1 이 정답인가
|
||
if v4['mdx_sections'][sid]['judgments_full32']:
|
||
base_top1_fn = sorted(v4['mdx_sections'][sid]['judgments_full32'], key=lambda x: -x['confidence'])[0]['frame_number']
|
||
if base_top1_fn == ans_fn:
|
||
correct_baseline += 1
|
||
abl_top1_fn = ablated_sorted[0]['fn']
|
||
if abl_top1_fn == ans_fn:
|
||
correct_ablated += 1
|
||
|
||
change = '동일' if (ab['label'] == aa['ablated_label'] and base_rank == abl_rank) else '⚠️ 변화'
|
||
md.append(f'| {sid} | Frame {ans_fn} | '
|
||
f'{ab["confidence"]:.3f} / {ab["label"]} / 순위 {base_rank} | '
|
||
f'{aa["ablated_conf"]:.3f} / {aa["ablated_label"]} / 순위 {abl_rank} | {change} |\n')
|
||
|
||
md.append(f'\n**TARGET Top-1 정답률**: Baseline {correct_baseline}/4, Ablated {correct_ablated}/4\n\n')
|
||
|
||
# ─── Top-3 변화 ───
|
||
md.append('## 결과 2 — 각 섹션 Top-3 매칭 변화\n')
|
||
for sid, r in section_results.items():
|
||
tag = 'TARGET' if r['is_target'] else 'Holdout'
|
||
ans = f'(정답 Frame {r["ans_fn"]})' if r['ans_fn'] else '(holdout)'
|
||
md.append(f'\n### [{sid}] {tag} {ans}\n')
|
||
md.append('| 순위 | Baseline | Ablated |\n|---|---|---|\n')
|
||
for i in range(3):
|
||
b = r['baseline_top3'][i] if i < len(r['baseline_top3']) else (None, 0, '')
|
||
a = r['ablated_top3'][i] if i < len(r['ablated_top3']) else (None, 0, '')
|
||
b_mark = ' 🎯' if r['ans_fn'] and b[0] == r['ans_fn'] else ''
|
||
a_mark = ' 🎯' if r['ans_fn'] and a[0] == r['ans_fn'] else ''
|
||
md.append(f'| {i+1} | Frame {b[0]}{b_mark} ({b[1]:.3f}, {b[2]}) | Frame {a[0]}{a_mark} ({a[1]:.3f}, {a[2]}) |\n')
|
||
|
||
# ─── 라벨 분포 변화 ───
|
||
md.append('\n## 결과 3 — 라벨 분포 변화 (32 frame 기준)\n')
|
||
md.append('| MDX | use_as_is | light_edit | restructure | reject |\n|---|---|---|---|---|\n')
|
||
for sid, r in section_results.items():
|
||
b = r['base_lbl']; a = r['abl_lbl']
|
||
def fmt(lbl):
|
||
bv = b.get(lbl, 0); av = a.get(lbl, 0)
|
||
if bv == av: return f'{bv}'
|
||
return f'{bv}→{av}'
|
||
md.append(f'| {sid} | {fmt("use_as_is")} | {fmt("light_edit")} | {fmt("restructure")} | {fmt("reject")} |\n')
|
||
|
||
# ─── 결론 ───
|
||
md.append('\n## 결론\n')
|
||
same_top1 = sum(1 for sid, r in section_results.items()
|
||
if r['baseline_top3'] and r['ablated_top3']
|
||
and r['baseline_top3'][0][0] == r['ablated_top3'][0][0])
|
||
same_top3 = sum(1 for sid, r in section_results.items()
|
||
if r['baseline_top3'] and r['ablated_top3']
|
||
and set(x[0] for x in r['baseline_top3']) == set(x[0] for x in r['ablated_top3']))
|
||
md.append(f'- **Top-1 일치 섹션 수**: {same_top1}/{len(section_results)}\n')
|
||
md.append(f'- **Top-3 멤버 일치 섹션 수**: {same_top3}/{len(section_results)}\n')
|
||
md.append(f'- **TARGET Top-1 정답률**: Baseline {correct_baseline}/4 → Ablated {correct_ablated}/4\n')
|
||
|
||
OUT_PATH.write_text(''.join(md), encoding='utf-8')
|
||
|
||
print('=' * 70)
|
||
print(f'V4 slot 축 ablation 완료: {OUT_PATH}')
|
||
print('=' * 70)
|
||
print(f'TARGET Top-1 정답률: Baseline {correct_baseline}/4, Ablated {correct_ablated}/4')
|
||
print(f'Top-1 동일 섹션: {same_top1}/{len(section_results)}')
|
||
print(f'Top-3 멤버 동일 섹션: {same_top3}/{len(section_results)}')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|