C.E.L_Slide_test2/tests/matching/pipeline_18_slot_axis_ablation.py

"""Pipeline Step 18 — V4 slot 축 ablation (실증).

질문: V4 의 5축 중 slot 축 (W_SLOT=0.15) 이 frame 매칭에 실제로 기여하는가?

방법:
  1. 기존 v4_full32_result (W_SLOT=0.15 유지) 를 baseline 으로
  2. W_SLOT=0 으로 두고 나머지 4축 (anchor/cardinality/relation/content) 가중치
     재정규화 — sum=1.0 유지 → ablated 결과 산출
  3. 32 frame × 7 MDX 섹션 confidence 재계산
  4. 비교:
     - TARGET 4개의 정답 frame 라벨/순위 변화
     - Top-3 매칭 변화
     - 라벨 분포 변화 (use_as_is/light_edit/restructure/reject 카운트)
  5. 결론: slot 축이 의미 있나?

출력: V4_SLOT_ABLATION.md
"""
import sys
from pathlib import Path
import yaml

HERE = Path(__file__).parent
sys.path.insert(0, str(HERE))

from template_fit import route

OUT_PATH = HERE / 'V4_SLOT_ABLATION.md'

# Baseline (현재): 0.25 + 0.20 + 0.20 + 0.15 + 0.20 = 1.00
W_BASELINE = {'anchor': 0.25, 'cardinality': 0.20, 'relation': 0.20, 'slot': 0.15, 'content': 0.20}

# Ablated: slot=0, 나머지 4축 비례 재정규화 → sum=1.00
# 4축 합 = 0.85, 각 축 / 0.85
W_ABLATED = {
    'anchor':      0.25 / 0.85,   # 0.2941
    'cardinality': 0.20 / 0.85,   # 0.2353
    'relation':    0.20 / 0.85,   # 0.2353
    'slot':        0.0,
    'content':     0.20 / 0.85,   # 0.2353
}


def recompute_confidence(axes, weights, penalty):
    base = sum(weights[k] * axes.get(k, 0) for k in weights)
    return max(0.0, base - penalty)


def main():
    v4 = yaml.safe_load((HERE / 'v4_full32_result.yaml').read_text(encoding='utf-8'))
    answer_map = v4['meta']['answer_map']
    holdout = set(v4['meta']['holdout_sections'])

    md = []
    md.append('# V4 slot 축 ablation — 실증\n')
    md.append('## 질문\n')
    md.append('V4 confidence 계산식의 slot 축 (W_SLOT=0.15) 이 frame 매칭에 실제로 기여하는가?\n')
    md.append('## 방법\n')
    md.append('- **Baseline**: 5축 가중치 그대로 — 0.25 anchor + 0.20 card + 0.20 rel + 0.15 slot + 0.20 content\n')
    md.append('- **Ablated**: W_SLOT=0, 나머지 4축 비례 재정규화 (sum=1.0)\n')
    md.append('  - anchor: 0.294, cardinality: 0.235, relation: 0.235, content: 0.235\n')
    md.append('- 32 frame × 7 MDX 섹션 confidence 재계산 → label 재할당 → 비교\n\n')

    section_results = {}
    for sid, sec in v4['mdx_sections'].items():
        is_target = sid not in holdout
        ans_fn = answer_map.get(sid)

        # Baseline 결과 (이미 v4_full32 에 있음)
        baseline = sorted(sec['judgments_full32'], key=lambda x: -x['confidence'])

        # Ablated 결과 재계산
        ablated = []
        for j in sec['judgments_full32']:
            new_conf = recompute_confidence(j['axes'], W_ABLATED, j['penalty'])
            new_label = route(new_conf)
            ablated.append({
                'frame_number': j['frame_number'],
                'frame_id': j['frame_id'],
                'baseline_conf': j['confidence'],
                'baseline_label': j['label'],
                'ablated_conf': round(new_conf, 4),
                'ablated_label': new_label,
            })
        ablated.sort(key=lambda x: -x['ablated_conf'])

        # 정답 frame 변화
        ans_baseline = next((x for x in baseline if x['frame_number'] == ans_fn), None) if ans_fn else None
        ans_ablated = next((x for x in ablated if x['frame_number'] == ans_fn), None) if ans_fn else None

        # 순위 비교
        baseline_top3 = [(x['frame_number'], x['confidence'], x['label']) for x in baseline[:3]]
        ablated_top3 = [(x['frame_number'], x['ablated_conf'], x['ablated_label']) for x in ablated[:3]]

        # 라벨 분포 비교
        from collections import Counter
        base_lbl = Counter(x['label'] for x in baseline)
        abl_lbl = Counter(x['ablated_label'] for x in ablated)

        section_results[sid] = {
            'is_target': is_target,
            'ans_fn': ans_fn,
            'baseline_top3': baseline_top3,
            'ablated_top3': ablated_top3,
            'ans_baseline': ans_baseline,
            'ans_ablated': ans_ablated,
            'base_lbl': base_lbl,
            'abl_lbl': abl_lbl,
        }

    # ─── TARGET 정답 매칭 변화 ───
    md.append('## 결과 1 — TARGET 4 섹션의 정답 frame 라벨/순위 변화\n')
    md.append('| MDX | 정답 Frame | Baseline conf / 라벨 / 순위 | Ablated conf / 라벨 / 순위 | 변화 |\n')
    md.append('|---|---|---|---|---|\n')
    target_sids = [sid for sid in section_results if section_results[sid]['is_target']]
    correct_baseline = 0
    correct_ablated = 0
    for sid in target_sids:
        r = section_results[sid]
        ans_fn = r['ans_fn']
        ab = r['ans_baseline']
        aa = r['ans_ablated']
        base_rank = next((i+1 for i, x in enumerate(sorted(v4['mdx_sections'][sid]['judgments_full32'], key=lambda x: -x['confidence'])) if x['frame_number'] == ans_fn), '?')
        # ablated rank
        ablated_sorted = sorted(
            [{'fn': j['frame_number'], 'c': recompute_confidence(j['axes'], W_ABLATED, j['penalty'])}
             for j in v4['mdx_sections'][sid]['judgments_full32']],
            key=lambda x: -x['c']
        )
        abl_rank = next((i+1 for i, x in enumerate(ablated_sorted) if x['fn'] == ans_fn), '?')

        # 정답률 — baseline / ablated 의 Top-1 이 정답인가
        if v4['mdx_sections'][sid]['judgments_full32']:
            base_top1_fn = sorted(v4['mdx_sections'][sid]['judgments_full32'], key=lambda x: -x['confidence'])[0]['frame_number']
            if base_top1_fn == ans_fn:
                correct_baseline += 1
            abl_top1_fn = ablated_sorted[0]['fn']
            if abl_top1_fn == ans_fn:
                correct_ablated += 1

        change = '동일' if (ab['label'] == aa['ablated_label'] and base_rank == abl_rank) else '⚠️ 변화'
        md.append(f'| {sid} | Frame {ans_fn} | '
                  f'{ab["confidence"]:.3f} / {ab["label"]} / 순위 {base_rank} | '
                  f'{aa["ablated_conf"]:.3f} / {aa["ablated_label"]} / 순위 {abl_rank} | {change} |\n')

    md.append(f'\n**TARGET Top-1 정답률**: Baseline {correct_baseline}/4, Ablated {correct_ablated}/4\n\n')

    # ─── Top-3 변화 ───
    md.append('## 결과 2 — 각 섹션 Top-3 매칭 변화\n')
    for sid, r in section_results.items():
        tag = 'TARGET' if r['is_target'] else 'Holdout'
        ans = f'(정답 Frame {r["ans_fn"]})' if r['ans_fn'] else '(holdout)'
        md.append(f'\n### [{sid}] {tag} {ans}\n')
        md.append('| 순위 | Baseline | Ablated |\n|---|---|---|\n')
        for i in range(3):
            b = r['baseline_top3'][i] if i < len(r['baseline_top3']) else (None, 0, '')
            a = r['ablated_top3'][i] if i < len(r['ablated_top3']) else (None, 0, '')
            b_mark = ' 🎯' if r['ans_fn'] and b[0] == r['ans_fn'] else ''
            a_mark = ' 🎯' if r['ans_fn'] and a[0] == r['ans_fn'] else ''
            md.append(f'| {i+1} | Frame {b[0]}{b_mark} ({b[1]:.3f}, {b[2]}) | Frame {a[0]}{a_mark} ({a[1]:.3f}, {a[2]}) |\n')

    # ─── 라벨 분포 변화 ───
    md.append('\n## 결과 3 — 라벨 분포 변화 (32 frame 기준)\n')
    md.append('| MDX | use_as_is | light_edit | restructure | reject |\n|---|---|---|---|---|\n')
    for sid, r in section_results.items():
        b = r['base_lbl']; a = r['abl_lbl']
        def fmt(lbl):
            bv = b.get(lbl, 0); av = a.get(lbl, 0)
            if bv == av: return f'{bv}'
            return f'{bv}→{av}'
        md.append(f'| {sid} | {fmt("use_as_is")} | {fmt("light_edit")} | {fmt("restructure")} | {fmt("reject")} |\n')

    # ─── 결론 ───
    md.append('\n## 결론\n')
    same_top1 = sum(1 for sid, r in section_results.items()
                    if r['baseline_top3'] and r['ablated_top3']
                    and r['baseline_top3'][0][0] == r['ablated_top3'][0][0])
    same_top3 = sum(1 for sid, r in section_results.items()
                    if r['baseline_top3'] and r['ablated_top3']
                    and set(x[0] for x in r['baseline_top3']) == set(x[0] for x in r['ablated_top3']))
    md.append(f'- **Top-1 일치 섹션 수**: {same_top1}/{len(section_results)}\n')
    md.append(f'- **Top-3 멤버 일치 섹션 수**: {same_top3}/{len(section_results)}\n')
    md.append(f'- **TARGET Top-1 정답률**: Baseline {correct_baseline}/4 → Ablated {correct_ablated}/4\n')

    OUT_PATH.write_text(''.join(md), encoding='utf-8')

    print('=' * 70)
    print(f'V4 slot 축 ablation 완료: {OUT_PATH}')
    print('=' * 70)
    print(f'TARGET Top-1 정답률: Baseline {correct_baseline}/4, Ablated {correct_ablated}/4')
    print(f'Top-1 동일 섹션: {same_top1}/{len(section_results)}')
    print(f'Top-3 멤버 동일 섹션: {same_top3}/{len(section_results)}')


if __name__ == '__main__':
    main()