C.E.L_Slide_test2/tests/matching/pipeline_08_v3_structure_rerank.py

"""Pipeline Step 8 — V3: MDX 구조 × Figma 구조 rerank

파이프라인 위치:
  V1 → V2 → **V3 (이 스크립트)** → V4

설계 원칙:
  - V2 결과(v2_semantic_rerank_result.yaml)의 Top-K 를 그대로 사용.
  - MDX 구조는 코드가 파악 (phase_common.detect_mdx_layout_v2).
  - Figma 구조는 AI가 이미 만들어 놓은 산출물(structure_ontology.yaml.templates_v1) 재사용.
    - Figma 구조 라벨은 `source.original_layout` 사용 (phase_common._COMPAT 키 체계와 일치).
  - 구조 호환도(_COMPAT) 로 재정렬. 가중합 아님 — 순수 rerank.
  - V3 에서 새 AI 패스 없음.

입력:
  - v2_semantic_rerank_result.yaml
  - MDX_SECTIONS 원문
  - structure_ontology.yaml (templates_v1)

출력:
  - v3_structure_rerank_result.yaml
"""
import hashlib
import sys
import datetime
from pathlib import Path
import yaml

HERE = Path(__file__).parent
sys.path.insert(0, str(HERE))

from phase_common import detect_mdx_layout_v2, structural_match_v2
from pipeline_01_extract_nodes import MDX_SECTIONS, MDX_DIR

LOCK_SNAPSHOT_FILES = [
    'pipeline_08_v3_structure_rerank.py',
    'pipeline_08_v2_semantic_rerank.py',
    'phase_common.py',
    'structure_ontology.yaml',
    'v2_semantic_rerank_result.yaml',
]


def sha256_file(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def extract_mdx_raw(section_id: str) -> str:
    cfg = MDX_SECTIONS[section_id]
    p = MDX_DIR / cfg['file']
    lines = p.read_text(encoding='utf-8').split('\n')
    start_idx = None
    for i, ln in enumerate(lines):
        if ln.strip() == cfg['start'].strip():
            start_idx = i
            break
    if start_idx is None:
        raise ValueError(f"start_heading 찾지 못함: {cfg['start']!r}")
    end_idx = len(lines)
    if cfg.get('end_prefix'):
        for i in range(start_idx + 1, len(lines)):
            if lines[i].strip().startswith(cfg['end_prefix']):
                end_idx = i
                break
    return '\n'.join(lines[start_idx:end_idx])


def main():
    v2 = yaml.safe_load((HERE / 'v2_semantic_rerank_result.yaml').read_text(encoding='utf-8'))
    ontology = yaml.safe_load((HERE / 'structure_ontology.yaml').read_text(encoding='utf-8'))
    templates_v1 = ontology['templates_v1']

    out_sections = {}
    for sid, v2_sec in v2['mdx_sections'].items():
        raw = extract_mdx_raw(sid)
        mdx_layout = detect_mdx_layout_v2(raw)

        rerank = []
        for cand in v2_sec['v2_rerank']:
            fid = cand['frame_id']
            tpl = templates_v1.get(fid, {})
            # Figma 구조 라벨: source.original_layout 우선, 없으면 visual_pattern.layout
            fig_layout = None
            if tpl:
                fig_layout = tpl.get('source', {}).get('original_layout')
                if not fig_layout:
                    fig_layout = tpl.get('visual_pattern', {}).get('layout')
            compat = structural_match_v2(mdx_layout, fig_layout) if fig_layout else 0.0

            rerank.append({
                'frame_id': fid,
                'frame_number': cand['frame_number'],
                'v1_rank': cand['v1_rank'],
                'v2_rank': cand['v2_rank'],
                'v1_score': cand['v1_score'],
                'semantic_score': cand['semantic_score'],
                'fig_layout': fig_layout,
                'structure_compat': round(float(compat), 4),
            })

        # 재정렬: structure_compat 내림차순, tie-break 은 V2 rank 오름차순
        rerank.sort(key=lambda x: (-x['structure_compat'], x['v2_rank']))
        for new_rank, item in enumerate(rerank, start=1):
            item['v3_rank'] = new_rank

        out_sections[sid] = {
            'section_type': v2_sec.get('section_type'),
            'answer_frame_number': v2_sec.get('answer_frame_number'),
            'mdx_title': v2_sec['mdx_title'],
            'mdx_layout': mdx_layout,
            'v3_rerank': rerank,
        }

    lock = {
        'timestamp': datetime.datetime.now().isoformat(timespec='seconds'),
        'top_k': v2['meta']['top_k'],
        'figma_structure_source': 'structure_ontology.yaml::templates_v1.<fid>.source.original_layout',
        'mdx_structure_source': 'phase_common.detect_mdx_layout_v2',
        'compat_table': 'phase_common._COMPAT (legacy)',
        'files': {name: sha256_file(HERE / name) for name in LOCK_SNAPSHOT_FILES},
        'principle': [
            'V2 후보 집합 유지 — Top-K 내에서만 재정렬',
            'MDX 구조는 코드 추출, Figma 구조는 기존 AI 산출 재사용',
            'Holdout 성적을 설계 근거로 사용하지 않음',
        ],
    }

    out = {
        'meta': {
            'pipeline_step': '8.v3',
            'description': 'V2 후보를 MDX 구조 × Figma 구조 호환도로 재정렬 (캐스케이드 rerank 2단)',
            'top_k': v2['meta']['top_k'],
            'holdout_sections': v2['meta']['holdout_sections'],
            'answer_map': v2['meta']['answer_map'],
            'lock_snapshot': lock,
        },
        'mdx_sections': out_sections,
    }

    out_path = HERE / 'v3_structure_rerank_result.yaml'
    out_path.write_text(
        yaml.safe_dump(out, allow_unicode=True, sort_keys=False, width=1000),
        encoding='utf-8',
    )

    print("=" * 70)
    print(f"V3 재정렬 완료: {out_path}")
    print("=" * 70)
    answer_map = v2['meta']['answer_map']
    for sid, s in out_sections.items():
        v2_top1 = next(r for r in s['v3_rerank'] if r['v2_rank'] == 1)['frame_number']
        v3_top1 = s['v3_rerank'][0]['frame_number']
        v3_top1_compat = s['v3_rerank'][0]['structure_compat']
        mdx_lo = s['mdx_layout']
        fig_lo = s['v3_rerank'][0]['fig_layout']
        answer_num = answer_map.get(sid)
        mark = ''
        if answer_num is not None:
            v3_ok = '✓' if v3_top1 == answer_num else '✗'
            mark = f"  정답={answer_num}  V3{v3_ok}"
        else:
            mark = "  (holdout)"
        print(f"  [{sid:8}] MDX={mdx_lo:14} V2 top1=Frame {v2_top1:>2} → V3 top1=Frame {v3_top1:>2} (Fig={fig_lo}, compat={v3_top1_compat}){mark}")
    print()


if __name__ == '__main__':
    main()