"""Pipeline Step 18 — V4 slot 축 ablation (실증). 질문: V4 의 5축 중 slot 축 (W_SLOT=0.15) 이 frame 매칭에 실제로 기여하는가? 방법: 1. 기존 v4_full32_result (W_SLOT=0.15 유지) 를 baseline 으로 2. W_SLOT=0 으로 두고 나머지 4축 (anchor/cardinality/relation/content) 가중치 재정규화 — sum=1.0 유지 → ablated 결과 산출 3. 32 frame × 7 MDX 섹션 confidence 재계산 4. 비교: - TARGET 4개의 정답 frame 라벨/순위 변화 - Top-3 매칭 변화 - 라벨 분포 변화 (use_as_is/light_edit/restructure/reject 카운트) 5. 결론: slot 축이 의미 있나? 출력: V4_SLOT_ABLATION.md """ import sys from pathlib import Path import yaml HERE = Path(__file__).parent sys.path.insert(0, str(HERE)) from template_fit import route OUT_PATH = HERE / 'V4_SLOT_ABLATION.md' # Baseline (현재): 0.25 + 0.20 + 0.20 + 0.15 + 0.20 = 1.00 W_BASELINE = {'anchor': 0.25, 'cardinality': 0.20, 'relation': 0.20, 'slot': 0.15, 'content': 0.20} # Ablated: slot=0, 나머지 4축 비례 재정규화 → sum=1.00 # 4축 합 = 0.85, 각 축 / 0.85 W_ABLATED = { 'anchor': 0.25 / 0.85, # 0.2941 'cardinality': 0.20 / 0.85, # 0.2353 'relation': 0.20 / 0.85, # 0.2353 'slot': 0.0, 'content': 0.20 / 0.85, # 0.2353 } def recompute_confidence(axes, weights, penalty): base = sum(weights[k] * axes.get(k, 0) for k in weights) return max(0.0, base - penalty) def main(): v4 = yaml.safe_load((HERE / 'v4_full32_result.yaml').read_text(encoding='utf-8')) answer_map = v4['meta']['answer_map'] holdout = set(v4['meta']['holdout_sections']) md = [] md.append('# V4 slot 축 ablation — 실증\n') md.append('## 질문\n') md.append('V4 confidence 계산식의 slot 축 (W_SLOT=0.15) 이 frame 매칭에 실제로 기여하는가?\n') md.append('## 방법\n') md.append('- **Baseline**: 5축 가중치 그대로 — 0.25 anchor + 0.20 card + 0.20 rel + 0.15 slot + 0.20 content\n') md.append('- **Ablated**: W_SLOT=0, 나머지 4축 비례 재정규화 (sum=1.0)\n') md.append(' - anchor: 0.294, cardinality: 0.235, relation: 0.235, content: 0.235\n') md.append('- 32 frame × 7 MDX 섹션 confidence 재계산 → label 재할당 → 비교\n\n') section_results = {} for sid, sec in v4['mdx_sections'].items(): is_target = sid not in holdout ans_fn = answer_map.get(sid) # Baseline 결과 (이미 v4_full32 에 있음) baseline = sorted(sec['judgments_full32'], key=lambda x: -x['confidence']) # Ablated 결과 재계산 ablated = [] for j in sec['judgments_full32']: new_conf = recompute_confidence(j['axes'], W_ABLATED, j['penalty']) new_label = route(new_conf) ablated.append({ 'frame_number': j['frame_number'], 'frame_id': j['frame_id'], 'baseline_conf': j['confidence'], 'baseline_label': j['label'], 'ablated_conf': round(new_conf, 4), 'ablated_label': new_label, }) ablated.sort(key=lambda x: -x['ablated_conf']) # 정답 frame 변화 ans_baseline = next((x for x in baseline if x['frame_number'] == ans_fn), None) if ans_fn else None ans_ablated = next((x for x in ablated if x['frame_number'] == ans_fn), None) if ans_fn else None # 순위 비교 baseline_top3 = [(x['frame_number'], x['confidence'], x['label']) for x in baseline[:3]] ablated_top3 = [(x['frame_number'], x['ablated_conf'], x['ablated_label']) for x in ablated[:3]] # 라벨 분포 비교 from collections import Counter base_lbl = Counter(x['label'] for x in baseline) abl_lbl = Counter(x['ablated_label'] for x in ablated) section_results[sid] = { 'is_target': is_target, 'ans_fn': ans_fn, 'baseline_top3': baseline_top3, 'ablated_top3': ablated_top3, 'ans_baseline': ans_baseline, 'ans_ablated': ans_ablated, 'base_lbl': base_lbl, 'abl_lbl': abl_lbl, } # ─── TARGET 정답 매칭 변화 ─── md.append('## 결과 1 — TARGET 4 섹션의 정답 frame 라벨/순위 변화\n') md.append('| MDX | 정답 Frame | Baseline conf / 라벨 / 순위 | Ablated conf / 라벨 / 순위 | 변화 |\n') md.append('|---|---|---|---|---|\n') target_sids = [sid for sid in section_results if section_results[sid]['is_target']] correct_baseline = 0 correct_ablated = 0 for sid in target_sids: r = section_results[sid] ans_fn = r['ans_fn'] ab = r['ans_baseline'] aa = r['ans_ablated'] base_rank = next((i+1 for i, x in enumerate(sorted(v4['mdx_sections'][sid]['judgments_full32'], key=lambda x: -x['confidence'])) if x['frame_number'] == ans_fn), '?') # ablated rank ablated_sorted = sorted( [{'fn': j['frame_number'], 'c': recompute_confidence(j['axes'], W_ABLATED, j['penalty'])} for j in v4['mdx_sections'][sid]['judgments_full32']], key=lambda x: -x['c'] ) abl_rank = next((i+1 for i, x in enumerate(ablated_sorted) if x['fn'] == ans_fn), '?') # 정답률 — baseline / ablated 의 Top-1 이 정답인가 if v4['mdx_sections'][sid]['judgments_full32']: base_top1_fn = sorted(v4['mdx_sections'][sid]['judgments_full32'], key=lambda x: -x['confidence'])[0]['frame_number'] if base_top1_fn == ans_fn: correct_baseline += 1 abl_top1_fn = ablated_sorted[0]['fn'] if abl_top1_fn == ans_fn: correct_ablated += 1 change = '동일' if (ab['label'] == aa['ablated_label'] and base_rank == abl_rank) else '⚠️ 변화' md.append(f'| {sid} | Frame {ans_fn} | ' f'{ab["confidence"]:.3f} / {ab["label"]} / 순위 {base_rank} | ' f'{aa["ablated_conf"]:.3f} / {aa["ablated_label"]} / 순위 {abl_rank} | {change} |\n') md.append(f'\n**TARGET Top-1 정답률**: Baseline {correct_baseline}/4, Ablated {correct_ablated}/4\n\n') # ─── Top-3 변화 ─── md.append('## 결과 2 — 각 섹션 Top-3 매칭 변화\n') for sid, r in section_results.items(): tag = 'TARGET' if r['is_target'] else 'Holdout' ans = f'(정답 Frame {r["ans_fn"]})' if r['ans_fn'] else '(holdout)' md.append(f'\n### [{sid}] {tag} {ans}\n') md.append('| 순위 | Baseline | Ablated |\n|---|---|---|\n') for i in range(3): b = r['baseline_top3'][i] if i < len(r['baseline_top3']) else (None, 0, '') a = r['ablated_top3'][i] if i < len(r['ablated_top3']) else (None, 0, '') b_mark = ' 🎯' if r['ans_fn'] and b[0] == r['ans_fn'] else '' a_mark = ' 🎯' if r['ans_fn'] and a[0] == r['ans_fn'] else '' md.append(f'| {i+1} | Frame {b[0]}{b_mark} ({b[1]:.3f}, {b[2]}) | Frame {a[0]}{a_mark} ({a[1]:.3f}, {a[2]}) |\n') # ─── 라벨 분포 변화 ─── md.append('\n## 결과 3 — 라벨 분포 변화 (32 frame 기준)\n') md.append('| MDX | use_as_is | light_edit | restructure | reject |\n|---|---|---|---|---|\n') for sid, r in section_results.items(): b = r['base_lbl']; a = r['abl_lbl'] def fmt(lbl): bv = b.get(lbl, 0); av = a.get(lbl, 0) if bv == av: return f'{bv}' return f'{bv}→{av}' md.append(f'| {sid} | {fmt("use_as_is")} | {fmt("light_edit")} | {fmt("restructure")} | {fmt("reject")} |\n') # ─── 결론 ─── md.append('\n## 결론\n') same_top1 = sum(1 for sid, r in section_results.items() if r['baseline_top3'] and r['ablated_top3'] and r['baseline_top3'][0][0] == r['ablated_top3'][0][0]) same_top3 = sum(1 for sid, r in section_results.items() if r['baseline_top3'] and r['ablated_top3'] and set(x[0] for x in r['baseline_top3']) == set(x[0] for x in r['ablated_top3'])) md.append(f'- **Top-1 일치 섹션 수**: {same_top1}/{len(section_results)}\n') md.append(f'- **Top-3 멤버 일치 섹션 수**: {same_top3}/{len(section_results)}\n') md.append(f'- **TARGET Top-1 정답률**: Baseline {correct_baseline}/4 → Ablated {correct_ablated}/4\n') OUT_PATH.write_text(''.join(md), encoding='utf-8') print('=' * 70) print(f'V4 slot 축 ablation 완료: {OUT_PATH}') print('=' * 70) print(f'TARGET Top-1 정답률: Baseline {correct_baseline}/4, Ablated {correct_ablated}/4') print(f'Top-1 동일 섹션: {same_top1}/{len(section_results)}') print(f'Top-3 멤버 동일 섹션: {same_top3}/{len(section_results)}') if __name__ == '__main__': main()