C.E.L_Slide_test2/tests/matching/clean_anchor_sets.py

"""Milestone 2 / Step 1: structure_ontology.yaml 의 anchor_sets 를 keyword_base 기준으로 청소.

1. keyword_base.yaml 로드 → canonical set + excluded set
2. structure_ontology.yaml templates_v1 의 모든 anchor_sets 순회
3. 각 anchor_set.terms 필터링:
   - canonical 에 있으면 유지
   - excluded 에 있으면 제거 + 로그
4. 특수 케이스: Frame 20 (1171281198) dx_sw_necessity — '필수성' → '필수' 로 대체
5. 제거 후 terms 가 비어버린 set 삭제 + 로그
6. 결과 저장 (backup: structure_ontology.yaml.pre_milestone2.bak)
7. CLEAN_ANCHOR_SETS_REPORT.md 변경사항 리포트 생성
"""
import sys
import shutil
from pathlib import Path
import yaml

sys.path.insert(0, str(Path(__file__).parent))

HERE = Path(__file__).parent


def load_keyword_base():
    path = HERE / "keyword_base.yaml"
    with open(path, encoding='utf-8') as f:
        data = yaml.safe_load(f)
    canonical = set(data['keywords'].keys())
    excluded = set(e['term'] for e in data['excluded'])
    # excluded 는 bucket 정보도 필요
    excluded_by_bucket = {}
    for e in data['excluded']:
        excluded_by_bucket.setdefault(e['bucket'], set()).add(e['term'])
    return canonical, excluded, excluded_by_bucket


def load_ontology():
    path = HERE / "structure_ontology.yaml"
    with open(path, encoding='utf-8') as f:
        return yaml.safe_load(f)


SPECIAL_REPLACEMENTS = {
    # Frame 20 (1171281198) 필수성 → 필수
    ('1171281198', 'dx_sw_necessity', '필수성'): '필수',
}


def clean_anchor_sets(ontology, canonical, excluded):
    """templates_v1 의 anchor_sets 를 keyword_base 기준으로 필터링."""
    templates = ontology.get('templates_v1', {})
    changes = []  # (frame_id, set_id, action, detail)

    for fid, tpl in templates.items():
        new_anchor_sets = []
        for s in tpl.get('anchor_sets', []):
            set_id = s.get('id', '?')
            original_terms = list(s.get('terms', []))
            kept_terms = []
            removed_terms = []
            replaced_terms = []

            for t in original_terms:
                special_key = (fid, set_id, t)
                if special_key in SPECIAL_REPLACEMENTS:
                    new_t = SPECIAL_REPLACEMENTS[special_key]
                    if new_t in canonical:
                        kept_terms.append(new_t)
                        replaced_terms.append((t, new_t))
                    else:
                        removed_terms.append(t)
                    continue

                if t in excluded:
                    removed_terms.append(t)
                elif t in canonical:
                    kept_terms.append(t)
                else:
                    # 분류 불명 - keyword_base 에 없음. 보수적으로 유지
                    kept_terms.append(t)
                    changes.append((fid, set_id, 'UNKNOWN_KEEP',
                                   f"'{t}' keyword_base 에 없으나 excluded 도 아님 — 유지"))

            if replaced_terms:
                for old, new in replaced_terms:
                    changes.append((fid, set_id, 'REPLACE', f"'{old}' → '{new}'"))
            if removed_terms:
                changes.append((fid, set_id, 'REMOVE', f"제거: {removed_terms}"))

            if not kept_terms:
                changes.append((fid, set_id, 'DROP_SET',
                               f'set 전체 삭제 (원본: {original_terms})'))
                continue

            # min_hits 재검증
            min_hits = s.get('min_hits', 1)
            if min_hits > len(kept_terms):
                new_min = len(kept_terms)
                changes.append((fid, set_id, 'ADJUST_MIN_HITS',
                               f'min_hits {min_hits} → {new_min} (terms 감소)'))
                s['min_hits'] = new_min

            s['terms'] = kept_terms
            new_anchor_sets.append(s)

        if len(new_anchor_sets) != len(tpl.get('anchor_sets', [])):
            removed_ids = [s.get('id', '?')
                          for s in tpl.get('anchor_sets', [])
                          if s.get('id', '?') not in [ns.get('id', '?') for ns in new_anchor_sets]]
            changes.append((fid, '*', 'TEMPLATE_ANCHOR_SETS_REDUCED',
                           f'{len(tpl["anchor_sets"])}→{len(new_anchor_sets)}, 삭제: {removed_ids}'))

        tpl['anchor_sets'] = new_anchor_sets

    return ontology, changes


def write_report(changes, excluded_by_bucket, canonical):
    lines = []
    lines.append("# Milestone 2 / Step 1 — anchor_sets 정리 리포트")
    lines.append("")
    lines.append("**목적**: `keyword_base.yaml` 기준으로 `structure_ontology.yaml` templates_v1 의 anchor_sets 정리.")
    lines.append("")
    lines.append(f"**canonical 수**: {len(canonical)}")
    lines.append("**excluded 수 (bucket 별):**")
    for bk, terms in sorted(excluded_by_bucket.items()):
        lines.append(f"- {bk}: {len(terms)}")
    lines.append("")

    # 변경사항 요약
    action_counts = {}
    for _, _, action, _ in changes:
        action_counts[action] = action_counts.get(action, 0) + 1
    lines.append("## 변경 요약")
    lines.append("")
    lines.append("| Action | 수 |")
    lines.append("|--------|----|")
    for action, cnt in sorted(action_counts.items()):
        lines.append(f"| {action} | {cnt} |")
    lines.append("")

    # frame 별 상세
    by_frame = {}
    for fid, set_id, action, detail in changes:
        by_frame.setdefault(fid, []).append((set_id, action, detail))

    lines.append(f"## Frame 별 변경 ({len(by_frame)}개 frame 영향)")
    lines.append("")
    for fid in sorted(by_frame.keys()):
        lines.append(f"### Frame {fid}")
        for set_id, action, detail in by_frame[fid]:
            lines.append(f"- **[{set_id}]** {action}: {detail}")
        lines.append("")

    out = HERE / "CLEAN_ANCHOR_SETS_REPORT.md"
    out.write_text("\n".join(lines), encoding='utf-8')
    print(f"리포트: {out}")


def main():
    print("[1] keyword_base 로드...")
    canonical, excluded, excluded_by_bucket = load_keyword_base()
    print(f"    canonical: {len(canonical)}, excluded: {len(excluded)}")

    print("[2] structure_ontology 로드...")
    ontology = load_ontology()

    print("[3] anchor_sets 청소...")
    ontology, changes = clean_anchor_sets(ontology, canonical, excluded)

    print(f"[4] 변경사항: {len(changes)}건")
    action_counts = {}
    for _, _, action, _ in changes:
        action_counts[action] = action_counts.get(action, 0) + 1
    for a, c in sorted(action_counts.items()):
        print(f"    {a}: {c}")

    print("[5] 백업 생성...")
    src = HERE / "structure_ontology.yaml"
    bak = HERE / "structure_ontology.yaml.pre_milestone2.bak"
    shutil.copy(src, bak)
    print(f"    {bak}")

    print("[6] 저장...")
    # yaml.safe_dump 는 anchor/alias 불필요, 순서 보존 위해 default_flow_style=False + sort_keys=False
    with open(src, 'w', encoding='utf-8') as f:
        yaml.safe_dump(ontology, f, allow_unicode=True, sort_keys=False, width=120)
    print(f"    {src}")

    print("[7] 리포트 작성...")
    write_report(changes, excluded_by_bucket, canonical)

    print("\n완료.")


if __name__ == "__main__":
    main()