C.E.L_Slide_test2/tests/pipeline/code/pipeline_04_normalize.py

"""Step 4 (4.1 반영): pre-Kiwi substitution + Kiwi user_dict 기반 canonical 토큰 정규화.

파이프라인:
  text_node
    → 4a-pre. PRE_COLLAPSE (regex): 디지털 전환(DX) / DX(DX) 계열 collapse
    → 4a. phrase_variants 치환 (긴 variant 먼저, canonical 재치환 금지)
    → 4b. Kiwi tokenize (user_dict: SL 8 + NNG 5)
    → 4c. NNG/NNP/SL/SN + 1글자/순수숫자 필터
    → canonical token list

설계 제약:
  - 긴 variant 먼저 치환
  - canonical 자기 자신 재치환 금지
  - DX canonical 하나로 통일 (디지털전환 canonical 없음)
  - 3D모델/2D도면 은 Step 4 에서 제외 (3D/2D 일반 차원 토큰 보존)
  - applied_replacements / pre_collapse_applied 카운트 분리 출력
  - replacement samples: per-group quota (beps 8 / frames 6 / mdx 6 = 20)
  - corpus 에 full token_frequency / token_frame_df / token_mdx_df 저장

산출:
  - normalized_text_tokens.yaml
  - replacement_report.yaml
"""
import re
from collections import Counter
from pathlib import Path
import yaml

# 정규화 로직은 keyword_normalizer 모듈에서 import (공유)
from keyword_normalizer import (
    USER_DICT_SL, USER_DICT_NNG, ALLOWED_TAGS,
    PRE_COLLAPSE, MAX_PAREN_LEN, PAREN_PATTERN, PAREN_SKIP_PATTERN, PAREN_SAMPLE_MAX,
    load_phrase_variants as _load_phrase_variants,
    build_substitutions, expand_parentheses, apply_substitutions,
    build_kiwi, extract_tokens,
)

HERE = Path(__file__).parent
INPUT_NODES = HERE / "actual_text_nodes.yaml"
SYNONYMS = HERE / "synonyms.yaml"
OUTPUT_TOKENS = HERE / "normalized_text_tokens.yaml"
OUTPUT_REPORT = HERE / "replacement_report.yaml"

# 로컬 wrapper (기존 호출 형식 유지: load_phrase_variants())
def load_phrase_variants():
    return _load_phrase_variants(SYNONYMS)


# replacement samples per-group quota (합 20)
SAMPLE_QUOTA = {'beps': 8, 'frames': 6, 'mdx': 6}


def dedup_keep_order(seq):
    seen = set()
    out = []
    for x in seq:
        if x in seen:
            continue
        seen.add(x)
        out.append(x)
    return out


def process_source(kiwi, subs, source_group, source_key, text_nodes,
                   replacement_counter, pre_collapse_counter,
                   replacement_samples, sample_count_by_group,
                   corpus_freq, paren_counter, paren_samples):
    raw = []
    for t in text_nodes:
        before = t
        after = apply_substitutions(t, subs, replacement_counter, pre_collapse_counter,
                                    paren_counter, paren_samples)
        if before != after and sample_count_by_group[source_group] < SAMPLE_QUOTA[source_group]:
            replacement_samples.append({
                'group': source_group,
                'source': source_key,
                'before': before,
                'after': after,
            })
            sample_count_by_group[source_group] += 1
        tokens = extract_tokens(kiwi, after)
        for tk in tokens:
            corpus_freq[tk] += 1
        raw.extend(tokens)
    unique = dedup_keep_order(raw)
    return {
        'raw_token_count': len(raw),
        'unique_token_count': len(unique),
        'unique_tokens': unique,
    }


def main():
    data = yaml.safe_load(INPUT_NODES.read_text(encoding='utf-8'))
    phrase_variants = load_phrase_variants()
    subs = build_substitutions(phrase_variants)
    kiwi = build_kiwi()

    replacement_counter = Counter()
    pre_collapse_counter = Counter()
    replacement_samples = []
    sample_count_by_group = {'beps': 0, 'frames': 0, 'mdx': 0}
    corpus_freq = Counter()
    paren_counter = Counter()
    paren_samples = []

    output = {
        'meta': {
            'pipeline_step': 4.2,
            'description': 'PRE_COLLAPSE + pre-Kiwi substitution + user_dict. phrase_variants 기준.',
            'user_dict': {'SL': USER_DICT_SL, 'NNG': USER_DICT_NNG},
            'phrase_variants_count': len(phrase_variants),
            'substitution_rule_count': len(subs),
            'pre_collapse_rule_count': len(PRE_COLLAPSE),
            'sample_quota': SAMPLE_QUOTA,
            'notes': [
                'DX canonical 유지 (디지털전환은 DX variant 로 통합)',
                '3D모델/2D도면 canonical 은 Step 4 에서 제외 (3D/2D 일반 차원 토큰 보존)',
                'PRE_COLLAPSE 는 regex 로 "디지털 전환(DX)" / "DX(DX)" 계열 통합',
            ],
        },
        'beps': {}, 'frames': {}, 'mdx': {}, 'corpus': {},
    }

    beps = data.get('beps') or {}
    if beps.get('text_nodes'):
        key = f"beps:{beps['frame_id']}"
        entry = process_source(kiwi, subs, 'beps', key, beps['text_nodes'],
                               replacement_counter, pre_collapse_counter,
                               replacement_samples, sample_count_by_group, corpus_freq,
                               paren_counter, paren_samples)
        output['beps'] = {'frame_id': beps['frame_id'], **entry}

    for fid, info in data.get('frames', {}).items():
        key = f"frame:{fid}"
        entry = process_source(kiwi, subs, 'frames', key, info.get('text_nodes', []),
                               replacement_counter, pre_collapse_counter,
                               replacement_samples, sample_count_by_group, corpus_freq,
                               paren_counter, paren_samples)
        output['frames'][fid] = entry

    for n, info in data.get('mdx', {}).items():
        key = f"mdx:{n}"
        entry = process_source(kiwi, subs, 'mdx', key, info.get('text_nodes', []),
                               replacement_counter, pre_collapse_counter,
                               replacement_samples, sample_count_by_group, corpus_freq,
                               paren_counter, paren_samples)
        output['mdx'][n] = entry

    # corpus 통계
    total_frames = len(output['frames'])
    frame_df = Counter()
    mdx_df = Counter()
    for fid, info in output['frames'].items():
        for tk in info['unique_tokens']:
            frame_df[tk] += 1
    for n, info in output['mdx'].items():
        for tk in info['unique_tokens']:
            mdx_df[tk] += 1

    top50_by_freq = [
        {'token': t, 'count': c,
         'frame_count': frame_df.get(t, 0), 'mdx_count': mdx_df.get(t, 0)}
        for t, c in corpus_freq.most_common(50)
    ]
    top50_by_frame_df = sorted(
        [{'token': t, 'frame_count': frame_df[t],
          'count': corpus_freq[t], 'mdx_count': mdx_df.get(t, 0)}
         for t in frame_df],
        key=lambda x: (-x['frame_count'], -x['count'], x['token']),
    )[:50]

    output['corpus'] = {
        'unique_token_count': len(corpus_freq),
        'total_occurrences': sum(corpus_freq.values()),
        'top_50_by_frequency': top50_by_freq,
        'top_50_by_frame_df': top50_by_frame_df,
        'token_frequency': dict(sorted(corpus_freq.items(), key=lambda x: (-x[1], x[0]))),
        'token_frame_df':  dict(sorted(frame_df.items(),    key=lambda x: (-x[1], x[0]))),
        'token_mdx_df':    dict(sorted(mdx_df.items(),      key=lambda x: (-x[1], x[0]))),
    }

    output['meta']['totals'] = {
        'beps_raw': output['beps'].get('raw_token_count', 0),
        'beps_unique': output['beps'].get('unique_token_count', 0),
        'frames_raw': sum(f['raw_token_count'] for f in output['frames'].values()),
        'frames_unique_avg': (
            sum(f['unique_token_count'] for f in output['frames'].values()) / total_frames
            if total_frames else 0
        ),
        'mdx_raw': sum(m['raw_token_count'] for m in output['mdx'].values()),
        'mdx_unique_avg': (
            sum(m['unique_token_count'] for m in output['mdx'].values()) / len(output['mdx'])
            if output['mdx'] else 0
        ),
        'corpus_unique': len(corpus_freq),
    }

    OUTPUT_TOKENS.write_text(
        yaml.safe_dump(output, allow_unicode=True, sort_keys=False, width=200),
        encoding='utf-8',
    )

    pre_collapse_total = sum(pre_collapse_counter.values())
    phrase_total = sum(replacement_counter.values())
    total_text_changes = pre_collapse_total + phrase_total

    report = {
        'meta': {
            'pipeline_step': 4.2,
            'description': 'PRE_COLLAPSE + 괄호 확장 + phrase_variants 치환 카운트 + per-group 쿼터 샘플',
            'pre_collapse_rule_count': len(PRE_COLLAPSE),
            'substitution_rule_count': len(subs),
            'sample_quota': SAMPLE_QUOTA,
            'paren_expansion_max_len': MAX_PAREN_LEN,
        },
        'pre_collapse_applied': dict(
            sorted(pre_collapse_counter.items(), key=lambda x: -x[1])
        ),
        'pre_collapse_total': pre_collapse_total,
        'paren_expanded': {
            'total': paren_counter.get('expansions', 0),
            'samples_count': len(paren_samples),
            'samples': paren_samples,
        },
        'applied_replacements': dict(
            sorted(replacement_counter.items(), key=lambda x: -x[1])
        ),
        'total_replacements': phrase_total,
        'total_text_changes': total_text_changes,
        'sample_count_by_group': sample_count_by_group,
        'samples': replacement_samples,
    }
    OUTPUT_REPORT.write_text(
        yaml.safe_dump(report, allow_unicode=True, sort_keys=False, width=200),
        encoding='utf-8',
    )

    # 콘솔 요약
    t = output['meta']['totals']
    print(f"[Step 4.2] PRE_COLLAPSE (4 rules) + pre-Kiwi substitution + user_dict")
    print(f"  PRE_COLLAPSE: {len(PRE_COLLAPSE)} regex rules")
    print(f"  phrase_variants: {len(phrase_variants)}개 canonical, {len(subs)}개 rule")
    print(f"  user_dict: SL {len(USER_DICT_SL)}, NNG {len(USER_DICT_NNG)}")
    print()
    print(f"  BEPS:   raw={t['beps_raw']}, unique={t['beps_unique']}")
    print(f"  Frames: raw={t['frames_raw']}, unique_avg={t['frames_unique_avg']:.1f} ({total_frames}개 frame)")
    print(f"  MDX:    raw={t['mdx_raw']}, unique_avg={t['mdx_unique_avg']:.1f}")
    print(f"  Corpus: unique={t['corpus_unique']}, occurrences={sum(corpus_freq.values())}")
    print()
    print(f"  pre_collapse_total:    {pre_collapse_total}")
    for rid, cnt in report['pre_collapse_applied'].items():
        print(f"    [{rid}] {cnt}")
    print(f"  applied_replacements total: {phrase_total}")
    for canonical, cnt in report['applied_replacements'].items():
        print(f"    {canonical:10s} {cnt}")
    print(f"  total_text_changes:    {total_text_changes}")
    print()
    print(f"  samples per group: {sample_count_by_group} (quota={SAMPLE_QUOTA})")
    print()
    print(f"산출: {OUTPUT_TOKENS}")
    print(f"      {OUTPUT_REPORT}")


if __name__ == "__main__":
    main()