"""Step 5.1: anchor 후보 랭킹 — 3그룹 + Full candidate. 변경점 (5.0 → 5.1): - top 15 → top 20 (frequent 그룹만) - 후보를 3 그룹 + full 로 분리: 1. frequent_candidates — frame_local_count 내림차순 top 20 2. special_candidates — USER_DICT_SL/NNG 중 해당 frame 에 등장한 것 (local desc) 3. unique_to_frame_candidates — frame_df=1 후보 (local desc) 4. full_candidates — 전체 frame 토큰 (local desc) - score 계산 없음. raw 지표만. 원칙: - AI 자동 추천 없음. 검수용 후보 목록. - 3그룹은 서로 겹칠 수 있음 (예: BIM 은 frequent + special 둘 다). 산출: - anchor_candidates.yaml (4개 리스트 구조) - anchor_candidates_report.md - anchor_candidates_report.html (details/summary, TARGET frames open) """ import math import sys from collections import Counter from pathlib import Path import yaml import markdown HERE = Path(__file__).parent sys.path.insert(0, str(HERE)) from pipeline_04_normalize import ( apply_substitutions, build_substitutions, build_kiwi, extract_tokens, USER_DICT_SL, USER_DICT_NNG, load_phrase_variants, ) INPUT_NODES = HERE / "actual_text_nodes.yaml" INPUT_NORMALIZED = HERE / "normalized_text_tokens.yaml" OUTPUT_YAML = HERE / "anchor_candidates.yaml" OUTPUT_MD = HERE / "anchor_candidates_report.md" OUTPUT_HTML = HERE / "anchor_candidates_report.html" SPECIAL_TOKENS = set(USER_DICT_SL) | set(USER_DICT_NNG) FREQUENT_TOP_N = 20 EXAMPLE_COUNT = 3 EXAMPLE_TRUNCATE = 80 TARGET_FRAMES = {13, 14, 18, 29} TOTAL_FRAMES = 32 def truncate(s, n): return s if len(s) <= n else s[:n - 1] + '…' def build_frame_token_counter(text_nodes, subs, kiwi): dummy_rep = Counter() dummy_pre = Counter() counter = Counter() normalized_texts = [] for t in text_nodes: after = apply_substitutions(t, subs, dummy_rep, dummy_pre) normalized_texts.append((t, after)) tokens = extract_tokens(kiwi, after) for tk in tokens: counter[tk] += 1 return counter, normalized_texts def find_examples(token, normalized_texts, limit=EXAMPLE_COUNT): seen = set() exs = [] for original, after in normalized_texts: if token in after and original not in seen: seen.add(original) exs.append(original) if len(exs) >= limit: break return exs def build_row(token, local, corpus_fd, corpus_md_df, normalized_texts): frame_df = corpus_fd.get(token, 0) mdx_df = corpus_md_df.get(token, 0) idf = round(math.log(TOTAL_FRAMES / frame_df), 3) if frame_df > 0 else 0.0 flags = [] if frame_df == 1: flags.append('unique_to_frame') return { 'token': token, 'frame_local_count': local, 'frame_df': frame_df, 'mdx_df': mdx_df, 'mdx_hit': mdx_df > 0, 'is_special': token in SPECIAL_TOKENS, 'idf': idf, 'flags': flags, 'examples': find_examples(token, normalized_texts), } def rank_groups_for_frame(text_nodes, subs, kiwi, corpus_fd, corpus_md_df): frame_counter, normalized_texts = build_frame_token_counter(text_nodes, subs, kiwi) all_rows = [ build_row(tok, local, corpus_fd, corpus_md_df, normalized_texts) for tok, local in frame_counter.items() ] # 정렬: local desc, idf desc, token asc all_rows.sort(key=lambda x: (-x['frame_local_count'], -x['idf'], x['token'])) frequent = all_rows[:FREQUENT_TOP_N] special = [r for r in all_rows if r['is_special']] unique_to_frame = [r for r in all_rows if 'unique_to_frame' in r['flags']] full = all_rows return { 'total_unique_tokens': len(all_rows), 'frequent_candidates': frequent, 'special_candidates': special, 'unique_to_frame_candidates': unique_to_frame, 'full_candidates': full, } # ---------- md / html 렌더 ---------- TABLE_HEADERS = ['token', 'local', 'df/32', 'mdx_df/3', 'mdx_hit', 'special', 'idf', 'flags', 'examples'] def format_examples_md(examples): if not examples: return '—' return '
'.join(truncate(e, EXAMPLE_TRUNCATE) for e in examples) def rows_to_md_table(rows): if not rows: return '_(없음)_' lines = ['| ' + ' | '.join(TABLE_HEADERS) + ' |', '|' + '|'.join(['---'] * len(TABLE_HEADERS)) + '|'] for c in rows: lines.append('| ' + ' | '.join([ c['token'], str(c['frame_local_count']), str(c['frame_df']), str(c['mdx_df']), 'Y' if c['mdx_hit'] else '', 'Y' if c['is_special'] else '', str(c['idf']), ', '.join(c['flags']), format_examples_md(c['examples']), ]) + ' |') return '\n'.join(lines) def frame_section_md(frame_number, frame_id, groups): head = f"### Frame {frame_number} / {frame_id} (total_unique {groups['total_unique_tokens']})" sub1 = f"#### 1. Frequent candidates — top {FREQUENT_TOP_N}\n\n" + rows_to_md_table(groups['frequent_candidates']) sub2 = f"#### 2. Special / canonical — {len(groups['special_candidates'])}개\n\n" + rows_to_md_table(groups['special_candidates']) sub3 = f"#### 3. Unique-to-frame (df=1) — {len(groups['unique_to_frame_candidates'])}개\n\n" + rows_to_md_table(groups['unique_to_frame_candidates']) sub4 = f"#### 4. Full candidates — 전체 {len(groups['full_candidates'])}개\n\n" + rows_to_md_table(groups['full_candidates']) return '\n\n'.join([head, sub1, sub2, sub3, sub4]) def frame_section_html(frame_number, frame_id, groups, open_by_default): head_text = f"Frame {frame_number} / {frame_id} (total_unique {groups['total_unique_tokens']})" def sub(label, rows): body_md = rows_to_md_table(rows) body_html = markdown.markdown(body_md, extensions=['tables']) return f"

{label}

\n{body_html}" sub1 = sub(f"1. Frequent candidates — top {FREQUENT_TOP_N}", groups['frequent_candidates']) sub2 = sub(f"2. Special / canonical — {len(groups['special_candidates'])}개", groups['special_candidates']) sub3 = sub(f"3. Unique-to-frame (df=1) — {len(groups['unique_to_frame_candidates'])}개", groups['unique_to_frame_candidates']) full_html = markdown.markdown(rows_to_md_table(groups['full_candidates']), extensions=['tables']) sub4 = (f"
4. Full candidates — 전체 " f"{len(groups['full_candidates'])}개 (접기/펼치기)\n" f"{full_html}\n
") opened = 'open' if open_by_default else '' return f"""
{head_text} {sub1} {sub2} {sub3} {sub4}
""" def main(): nodes = yaml.safe_load(INPUT_NODES.read_text(encoding='utf-8')) normalized = yaml.safe_load(INPUT_NORMALIZED.read_text(encoding='utf-8')) phrase_variants = load_phrase_variants() subs = build_substitutions(phrase_variants) kiwi = build_kiwi() corpus_fd = normalized['corpus']['token_frame_df'] corpus_md_df = normalized['corpus']['token_mdx_df'] frame_ids = sorted(nodes['frames'].keys()) output_yaml = { 'meta': { 'pipeline_step': 5.1, 'frequent_top_n': FREQUENT_TOP_N, 'total_frames': len(frame_ids), 'special_tokens': sorted(SPECIAL_TOKENS), 'sort_order': ['frame_local_count desc', 'idf desc', 'token asc'], 'groups': [ '1. frequent_candidates (top 20)', '2. special_candidates (USER_DICT 중 해당 frame 등장)', '3. unique_to_frame_candidates (frame_df=1)', '4. full_candidates (전체 frame token)', ], 'note': ( 'score 없음. raw 지표 + flags + examples. ' '3 그룹은 서로 겹칠 수 있음 (예: BIM 은 frequent + special 둘 다). ' 'AI 자동 추천 없음. 검수용 후보 목록.' ), }, 'frames': {}, } md_sections = [ "# Anchor Candidates Report — Step 5.1", "", f"- total frames: **{len(frame_ids)}**", f"- frequent top N: **{FREQUENT_TOP_N}**", f"- sort: frame_local_count desc → idf desc → token asc", f"- special tokens ({len(SPECIAL_TOKENS)}): {', '.join(sorted(SPECIAL_TOKENS))}", "", "## 그룹 설명", "1. **Frequent candidates** — frame 내 등장 횟수 top 20", "2. **Special / canonical** — USER_DICT_SL/NNG 중 해당 frame 에 등장한 것", "3. **Unique-to-frame** — frame_df=1 (그 frame 에만 등장)", "4. **Full candidates** — 전체 frame 토큰 (참조용)", "", "**score 없음. raw 지표만. 검수용 후보 목록.**", "", ] html_sections = [] for i, fid in enumerate(frame_ids, 1): info = nodes['frames'][fid] text_nodes = info.get('text_nodes', []) if not text_nodes: continue groups = rank_groups_for_frame(text_nodes, subs, kiwi, corpus_fd, corpus_md_df) output_yaml['frames'][fid] = { 'frame_number': i, 'total_unique_tokens': groups['total_unique_tokens'], 'frequent_candidates': groups['frequent_candidates'], 'special_candidates': groups['special_candidates'], 'unique_to_frame_candidates': groups['unique_to_frame_candidates'], 'full_candidates': groups['full_candidates'], } md_sections.append(frame_section_md(i, fid, groups)) md_sections.append("") html_sections.append( frame_section_html(i, fid, groups, open_by_default=(i in TARGET_FRAMES)) ) OUTPUT_YAML.write_text( yaml.safe_dump(output_yaml, allow_unicode=True, sort_keys=False, width=300), encoding='utf-8', ) OUTPUT_MD.write_text('\n'.join(md_sections), encoding='utf-8') html_header = f"""

Anchor Candidates Report — Step 5.1

그룹 설명

  1. Frequent candidates — frame 내 등장 횟수 top {FREQUENT_TOP_N}
  2. Special / canonical — USER_DICT 중 해당 frame 에 등장한 것
  3. Unique-to-frame — frame_df=1 (그 frame 에만 등장)
  4. Full candidates — 전체 frame 토큰 (참조용, 접기)

score 없음. raw 지표만. AI 자동 추천 없음. 검수용 후보 목록.

TARGET frames 13 / 14 / 18 / 29 는 펼쳐진 상태, 나머지는 접힘.

""" html = f""" Anchor Candidates Report — Step 5.1 {html_header} {''.join(html_sections)} """ OUTPUT_HTML.write_text(html, encoding='utf-8') print(f"[Step 5.1] 3그룹 + Full 구조 생성 완료") print(f" frames: {len(output_yaml['frames'])}") print(f" frequent_top_n: {FREQUENT_TOP_N}") print(f" special tokens: {len(SPECIAL_TOKENS)}") # 그룹별 크기 통계 special_sizes = [len(f['special_candidates']) for f in output_yaml['frames'].values()] uniq_sizes = [len(f['unique_to_frame_candidates']) for f in output_yaml['frames'].values()] full_sizes = [len(f['full_candidates']) for f in output_yaml['frames'].values()] print(f" special_candidates: avg={sum(special_sizes)/len(special_sizes):.1f} max={max(special_sizes)}") print(f" unique_to_frame: avg={sum(uniq_sizes)/len(uniq_sizes):.1f} max={max(uniq_sizes)}") print(f" full_candidates: avg={sum(full_sizes)/len(full_sizes):.1f} max={max(full_sizes)}") print() print(f" 산출:") print(f" yaml: {OUTPUT_YAML}") print(f" md: {OUTPUT_MD}") print(f" html: {OUTPUT_HTML}") if __name__ == "__main__": main()