C.E.L_Slide_test2/tests/matching/pipeline_05_anchor_candidates.py

"""Step 5.1: anchor 후보 랭킹 — 3그룹 + Full candidate.

변경점 (5.0 → 5.1):
- top 15 → top 20 (frequent 그룹만)
- 후보를 3 그룹 + full 로 분리:
    1. frequent_candidates — frame_local_count 내림차순 top 20
    2. special_candidates — USER_DICT_SL/NNG 중 해당 frame 에 등장한 것 (local desc)
    3. unique_to_frame_candidates — frame_df=1 후보 (local desc)
    4. full_candidates — 전체 frame 토큰 (local desc)
- score 계산 없음. raw 지표만.

원칙:
- AI 자동 추천 없음. 검수용 후보 목록.
- 3그룹은 서로 겹칠 수 있음 (예: BIM 은 frequent + special 둘 다).

산출:
  - anchor_candidates.yaml (4개 리스트 구조)
  - anchor_candidates_report.md
  - anchor_candidates_report.html (details/summary, TARGET frames open)
"""
import math
import sys
from collections import Counter
from pathlib import Path
import yaml
import markdown

HERE = Path(__file__).parent
sys.path.insert(0, str(HERE))

from pipeline_04_normalize import (
    apply_substitutions, build_substitutions, build_kiwi,
    extract_tokens, USER_DICT_SL, USER_DICT_NNG,
    load_phrase_variants,
)

INPUT_NODES = HERE / "actual_text_nodes.yaml"
INPUT_NORMALIZED = HERE / "normalized_text_tokens.yaml"
OUTPUT_YAML = HERE / "anchor_candidates.yaml"
OUTPUT_MD = HERE / "anchor_candidates_report.md"
OUTPUT_HTML = HERE / "anchor_candidates_report.html"

SPECIAL_TOKENS = set(USER_DICT_SL) | set(USER_DICT_NNG)
FREQUENT_TOP_N = 20
EXAMPLE_COUNT = 3
EXAMPLE_TRUNCATE = 80
TARGET_FRAMES = {13, 14, 18, 29}
TOTAL_FRAMES = 32


def truncate(s, n):
    return s if len(s) <= n else s[:n - 1] + '…'


def build_frame_token_counter(text_nodes, subs, kiwi):
    dummy_rep = Counter()
    dummy_pre = Counter()
    counter = Counter()
    normalized_texts = []
    for t in text_nodes:
        after = apply_substitutions(t, subs, dummy_rep, dummy_pre)
        normalized_texts.append((t, after))
        tokens = extract_tokens(kiwi, after)
        for tk in tokens:
            counter[tk] += 1
    return counter, normalized_texts


def find_examples(token, normalized_texts, limit=EXAMPLE_COUNT):
    seen = set()
    exs = []
    for original, after in normalized_texts:
        if token in after and original not in seen:
            seen.add(original)
            exs.append(original)
            if len(exs) >= limit:
                break
    return exs


def build_row(token, local, corpus_fd, corpus_md_df, normalized_texts):
    frame_df = corpus_fd.get(token, 0)
    mdx_df = corpus_md_df.get(token, 0)
    idf = round(math.log(TOTAL_FRAMES / frame_df), 3) if frame_df > 0 else 0.0
    flags = []
    if frame_df == 1:
        flags.append('unique_to_frame')
    return {
        'token': token,
        'frame_local_count': local,
        'frame_df': frame_df,
        'mdx_df': mdx_df,
        'mdx_hit': mdx_df > 0,
        'is_special': token in SPECIAL_TOKENS,
        'idf': idf,
        'flags': flags,
        'examples': find_examples(token, normalized_texts),
    }


def rank_groups_for_frame(text_nodes, subs, kiwi, corpus_fd, corpus_md_df):
    frame_counter, normalized_texts = build_frame_token_counter(text_nodes, subs, kiwi)
    all_rows = [
        build_row(tok, local, corpus_fd, corpus_md_df, normalized_texts)
        for tok, local in frame_counter.items()
    ]
    # 정렬: local desc, idf desc, token asc
    all_rows.sort(key=lambda x: (-x['frame_local_count'], -x['idf'], x['token']))

    frequent = all_rows[:FREQUENT_TOP_N]
    special = [r for r in all_rows if r['is_special']]
    unique_to_frame = [r for r in all_rows if 'unique_to_frame' in r['flags']]
    full = all_rows
    return {
        'total_unique_tokens': len(all_rows),
        'frequent_candidates': frequent,
        'special_candidates': special,
        'unique_to_frame_candidates': unique_to_frame,
        'full_candidates': full,
    }


# ---------- md / html 렌더 ----------

TABLE_HEADERS = ['token', 'local', 'df/32', 'mdx_df/3', 'mdx_hit',
                 'special', 'idf', 'flags', 'examples']


def format_examples_md(examples):
    if not examples:
        return '—'
    return '<br>'.join(truncate(e, EXAMPLE_TRUNCATE) for e in examples)


def rows_to_md_table(rows):
    if not rows:
        return '_(없음)_'
    lines = ['| ' + ' | '.join(TABLE_HEADERS) + ' |',
             '|' + '|'.join(['---'] * len(TABLE_HEADERS)) + '|']
    for c in rows:
        lines.append('| ' + ' | '.join([
            c['token'],
            str(c['frame_local_count']),
            str(c['frame_df']),
            str(c['mdx_df']),
            'Y' if c['mdx_hit'] else '',
            'Y' if c['is_special'] else '',
            str(c['idf']),
            ', '.join(c['flags']),
            format_examples_md(c['examples']),
        ]) + ' |')
    return '\n'.join(lines)


def frame_section_md(frame_number, frame_id, groups):
    head = f"### Frame {frame_number} / {frame_id}  (total_unique {groups['total_unique_tokens']})"
    sub1 = f"#### 1. Frequent candidates — top {FREQUENT_TOP_N}\n\n" + rows_to_md_table(groups['frequent_candidates'])
    sub2 = f"#### 2. Special / canonical — {len(groups['special_candidates'])}개\n\n" + rows_to_md_table(groups['special_candidates'])
    sub3 = f"#### 3. Unique-to-frame (df=1) — {len(groups['unique_to_frame_candidates'])}개\n\n" + rows_to_md_table(groups['unique_to_frame_candidates'])
    sub4 = f"#### 4. Full candidates — 전체 {len(groups['full_candidates'])}개\n\n" + rows_to_md_table(groups['full_candidates'])
    return '\n\n'.join([head, sub1, sub2, sub3, sub4])


def frame_section_html(frame_number, frame_id, groups, open_by_default):
    head_text = f"Frame {frame_number} / {frame_id}  (total_unique {groups['total_unique_tokens']})"

    def sub(label, rows):
        body_md = rows_to_md_table(rows)
        body_html = markdown.markdown(body_md, extensions=['tables'])
        return f"<h4>{label}</h4>\n{body_html}"

    sub1 = sub(f"1. Frequent candidates — top {FREQUENT_TOP_N}", groups['frequent_candidates'])
    sub2 = sub(f"2. Special / canonical — {len(groups['special_candidates'])}개",
               groups['special_candidates'])
    sub3 = sub(f"3. Unique-to-frame (df=1) — {len(groups['unique_to_frame_candidates'])}개",
               groups['unique_to_frame_candidates'])
    full_html = markdown.markdown(rows_to_md_table(groups['full_candidates']),
                                  extensions=['tables'])
    sub4 = (f"<details><summary><b>4. Full candidates — 전체 "
            f"{len(groups['full_candidates'])}개 (접기/펼치기)</b></summary>\n"
            f"{full_html}\n</details>")

    opened = 'open' if open_by_default else ''
    return f"""<details {opened}>
<summary><b>{head_text}</b></summary>
{sub1}
{sub2}
{sub3}
{sub4}
</details>"""


def main():
    nodes = yaml.safe_load(INPUT_NODES.read_text(encoding='utf-8'))
    normalized = yaml.safe_load(INPUT_NORMALIZED.read_text(encoding='utf-8'))
    phrase_variants = load_phrase_variants()
    subs = build_substitutions(phrase_variants)
    kiwi = build_kiwi()

    corpus_fd = normalized['corpus']['token_frame_df']
    corpus_md_df = normalized['corpus']['token_mdx_df']

    frame_ids = sorted(nodes['frames'].keys())

    output_yaml = {
        'meta': {
            'pipeline_step': 5.1,
            'frequent_top_n': FREQUENT_TOP_N,
            'total_frames': len(frame_ids),
            'special_tokens': sorted(SPECIAL_TOKENS),
            'sort_order': ['frame_local_count desc', 'idf desc', 'token asc'],
            'groups': [
                '1. frequent_candidates (top 20)',
                '2. special_candidates (USER_DICT 중 해당 frame 등장)',
                '3. unique_to_frame_candidates (frame_df=1)',
                '4. full_candidates (전체 frame token)',
            ],
            'note': (
                'score 없음. raw 지표 + flags + examples. '
                '3 그룹은 서로 겹칠 수 있음 (예: BIM 은 frequent + special 둘 다). '
                'AI 자동 추천 없음. 검수용 후보 목록.'
            ),
        },
        'frames': {},
    }

    md_sections = [
        "# Anchor Candidates Report — Step 5.1",
        "",
        f"- total frames: **{len(frame_ids)}**",
        f"- frequent top N: **{FREQUENT_TOP_N}**",
        f"- sort: frame_local_count desc → idf desc → token asc",
        f"- special tokens ({len(SPECIAL_TOKENS)}): {', '.join(sorted(SPECIAL_TOKENS))}",
        "",
        "## 그룹 설명",
        "1. **Frequent candidates** — frame 내 등장 횟수 top 20",
        "2. **Special / canonical** — USER_DICT_SL/NNG 중 해당 frame 에 등장한 것",
        "3. **Unique-to-frame** — frame_df=1 (그 frame 에만 등장)",
        "4. **Full candidates** — 전체 frame 토큰 (참조용)",
        "",
        "**score 없음. raw 지표만. 검수용 후보 목록.**",
        "",
    ]

    html_sections = []

    for i, fid in enumerate(frame_ids, 1):
        info = nodes['frames'][fid]
        text_nodes = info.get('text_nodes', [])
        if not text_nodes:
            continue
        groups = rank_groups_for_frame(text_nodes, subs, kiwi, corpus_fd, corpus_md_df)
        output_yaml['frames'][fid] = {
            'frame_number': i,
            'total_unique_tokens': groups['total_unique_tokens'],
            'frequent_candidates': groups['frequent_candidates'],
            'special_candidates': groups['special_candidates'],
            'unique_to_frame_candidates': groups['unique_to_frame_candidates'],
            'full_candidates': groups['full_candidates'],
        }
        md_sections.append(frame_section_md(i, fid, groups))
        md_sections.append("")
        html_sections.append(
            frame_section_html(i, fid, groups, open_by_default=(i in TARGET_FRAMES))
        )

    OUTPUT_YAML.write_text(
        yaml.safe_dump(output_yaml, allow_unicode=True, sort_keys=False, width=300),
        encoding='utf-8',
    )
    OUTPUT_MD.write_text('\n'.join(md_sections), encoding='utf-8')

    html_header = f"""<h1>Anchor Candidates Report — Step 5.1</h1>
<ul>
  <li>total frames: <b>{len(frame_ids)}</b></li>
  <li>frequent top N: <b>{FREQUENT_TOP_N}</b></li>
  <li>sort: frame_local_count desc → idf desc → token asc</li>
  <li>special tokens ({len(SPECIAL_TOKENS)}): {', '.join(sorted(SPECIAL_TOKENS))}</li>
</ul>
<h2>그룹 설명</h2>
<ol>
  <li><b>Frequent candidates</b> — frame 내 등장 횟수 top {FREQUENT_TOP_N}</li>
  <li><b>Special / canonical</b> — USER_DICT 중 해당 frame 에 등장한 것</li>
  <li><b>Unique-to-frame</b> — frame_df=1 (그 frame 에만 등장)</li>
  <li><b>Full candidates</b> — 전체 frame 토큰 (참조용, 접기)</li>
</ol>
<p><b>score 없음. raw 지표만. AI 자동 추천 없음. 검수용 후보 목록.</b></p>
<p>TARGET frames 13 / 14 / 18 / 29 는 펼쳐진 상태, 나머지는 접힘.</p>
<hr>
"""
    html = f"""<!DOCTYPE html>
<html lang="ko">
<head>
<meta charset="utf-8">
<title>Anchor Candidates Report — Step 5.1</title>
<style>
body {{ font-family: -apple-system, "Segoe UI", Pretendard, sans-serif; max-width: 1300px; margin: 2em auto; padding: 0 1em; line-height: 1.5; color: #222; }}
h1 {{ border-bottom: 2px solid #333; padding-bottom: 0.2em; }}
h2 {{ margin-top: 1.5em; }}
h3 {{ margin: 0; color: #333; }}
h4 {{ margin-top: 1.2em; margin-bottom: 0.4em; color: #555; border-bottom: 1px dashed #ccc; padding-bottom: 0.2em; }}
details {{ margin: 0.6em 0; border: 1px solid #ddd; border-radius: 4px; padding: 0.5em 1em; }}
details[open] > summary {{ border-bottom: 1px solid #eee; margin-bottom: 0.5em; padding-bottom: 0.3em; }}
details details {{ border: 1px dashed #ccc; background: #fcfcfc; }}
details[open] {{ background: #fafbfc; }}
summary {{ cursor: pointer; font-size: 1.02em; padding: 0.3em 0; color: #0a6; }}
summary b {{ color: #222; }}
table {{ border-collapse: collapse; margin: 0.3em 0 0.8em 0; font-size: 0.85em; width: 100%; }}
th, td {{ border: 1px solid #ddd; padding: 4px 8px; text-align: left; vertical-align: top; }}
th {{ background: #f4f4f4; }}
code {{ background: #f4f4f4; padding: 2px 4px; border-radius: 3px; }}
em {{ color: #888; }}
</style>
</head>
<body>
{html_header}
{''.join(html_sections)}
</body>
</html>"""
    OUTPUT_HTML.write_text(html, encoding='utf-8')

    print(f"[Step 5.1] 3그룹 + Full 구조 생성 완료")
    print(f"  frames: {len(output_yaml['frames'])}")
    print(f"  frequent_top_n: {FREQUENT_TOP_N}")
    print(f"  special tokens: {len(SPECIAL_TOKENS)}")

    # 그룹별 크기 통계
    special_sizes = [len(f['special_candidates']) for f in output_yaml['frames'].values()]
    uniq_sizes = [len(f['unique_to_frame_candidates']) for f in output_yaml['frames'].values()]
    full_sizes = [len(f['full_candidates']) for f in output_yaml['frames'].values()]
    print(f"  special_candidates:        avg={sum(special_sizes)/len(special_sizes):.1f}  max={max(special_sizes)}")
    print(f"  unique_to_frame:           avg={sum(uniq_sizes)/len(uniq_sizes):.1f}  max={max(uniq_sizes)}")
    print(f"  full_candidates:           avg={sum(full_sizes)/len(full_sizes):.1f}  max={max(full_sizes)}")
    print()
    print(f"  산출:")
    print(f"    yaml: {OUTPUT_YAML}")
    print(f"    md:   {OUTPUT_MD}")
    print(f"    html: {OUTPUT_HTML}")


if __name__ == "__main__":
    main()