wip: phase_z2 evidence 파이프라인 + matching 실험(phase2~26) + 프론트 trace 패널 진행분 스냅샷

- src: phase_z2 composition/mapper/pipeline/placement_planner/retry, ai_fallback(prompts/schema/validate), mdx_text_atoms 신규 - Front: PipelineTracePanel 신규, FramePanel/SlideCanvas/Home/designAgentApi 등 갱신 + 테스트 4종 추가 - templates/phase_z2: catalog(component_expansion_registry, node_slot_mapping 신규), frames, families, slide_base 갱신 - tests/matching: phase2~26 매칭 실험 스크립트·리포트·온톨로지 전체 (미커밋 진행분) - tests: b4_v4 evidence, task5~28.5 시리즈, regression(imp95 baseline) 등 신규 테스트 대량 추가 - docs/reference: MDX 구조 인벤토리, MDX→Frame 구조 계약 문서 - scripts: mdx 계약/parity/coverage/viewport 체크, gitea comment, run sync 유틸 - .gitignore: tmp*.json, chromedriver, .orchestrator, *.pkl, Front_test* 등 임시/스냅샷 제외 미완성 작업의 보존용 스냅샷 커밋 (2026-07-02) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-02 17:03:42 +09:00
parent 97b7833a1b
commit b836e79ee1
527 changed files with 673036 additions and 717 deletions
@@ -0,0 +1,266 @@
+"""Step 4 (4.1 반영): pre-Kiwi substitution + Kiwi user_dict 기반 canonical 토큰 정규화.
+
+파이프라인:
+  text_node
+    → 4a-pre. PRE_COLLAPSE (regex): 디지털 전환(DX) / DX(DX) 계열 collapse
+    → 4a. phrase_variants 치환 (긴 variant 먼저, canonical 재치환 금지)
+    → 4b. Kiwi tokenize (user_dict: SL 8 + NNG 5)
+    → 4c. NNG/NNP/SL/SN + 1글자/순수숫자 필터
+    → canonical token list
+
+설계 제약:
+  - 긴 variant 먼저 치환
+  - canonical 자기 자신 재치환 금지
+  - DX canonical 하나로 통일 (디지털전환 canonical 없음)
+  - 3D모델/2D도면 은 Step 4 에서 제외 (3D/2D 일반 차원 토큰 보존)
+  - applied_replacements / pre_collapse_applied 카운트 분리 출력
+  - replacement samples: per-group quota (beps 8 / frames 6 / mdx 6 = 20)
+  - corpus 에 full token_frequency / token_frame_df / token_mdx_df 저장
+
+산출:
+  - normalized_text_tokens.yaml
+  - replacement_report.yaml
+"""
+import re
+from collections import Counter
+from pathlib import Path
+import yaml
+
+# 정규화 로직은 keyword_normalizer 모듈에서 import (공유)
+from keyword_normalizer import (
+    USER_DICT_SL, USER_DICT_NNG, ALLOWED_TAGS,
+    PRE_COLLAPSE, MAX_PAREN_LEN, PAREN_PATTERN, PAREN_SKIP_PATTERN, PAREN_SAMPLE_MAX,
+    load_phrase_variants as _load_phrase_variants,
+    build_substitutions, expand_parentheses, apply_substitutions,
+    build_kiwi, extract_tokens,
+)
+
+HERE = Path(__file__).parent
+INPUT_NODES = HERE / "actual_text_nodes.yaml"
+SYNONYMS = HERE / "synonyms.yaml"
+OUTPUT_TOKENS = HERE / "normalized_text_tokens.yaml"
+OUTPUT_REPORT = HERE / "replacement_report.yaml"
+
+# 로컬 wrapper (기존 호출 형식 유지: load_phrase_variants())
+def load_phrase_variants():
+    return _load_phrase_variants(SYNONYMS)
+
+
+# replacement samples per-group quota (합 20)
+SAMPLE_QUOTA = {'beps': 8, 'frames': 6, 'mdx': 6}
+
+
+def dedup_keep_order(seq):
+    seen = set()
+    out = []
+    for x in seq:
+        if x in seen:
+            continue
+        seen.add(x)
+        out.append(x)
+    return out
+
+
+def process_source(kiwi, subs, source_group, source_key, text_nodes,
+                   replacement_counter, pre_collapse_counter,
+                   replacement_samples, sample_count_by_group,
+                   corpus_freq, paren_counter, paren_samples):
+    raw = []
+    for t in text_nodes:
+        before = t
+        after = apply_substitutions(t, subs, replacement_counter, pre_collapse_counter,
+                                    paren_counter, paren_samples)
+        if before != after and sample_count_by_group[source_group] < SAMPLE_QUOTA[source_group]:
+            replacement_samples.append({
+                'group': source_group,
+                'source': source_key,
+                'before': before,
+                'after': after,
+            })
+            sample_count_by_group[source_group] += 1
+        tokens = extract_tokens(kiwi, after)
+        for tk in tokens:
+            corpus_freq[tk] += 1
+        raw.extend(tokens)
+    unique = dedup_keep_order(raw)
+    return {
+        'raw_token_count': len(raw),
+        'unique_token_count': len(unique),
+        'unique_tokens': unique,
+    }
+
+
+def main():
+    data = yaml.safe_load(INPUT_NODES.read_text(encoding='utf-8'))
+    phrase_variants = load_phrase_variants()
+    subs = build_substitutions(phrase_variants)
+    kiwi = build_kiwi()
+
+    replacement_counter = Counter()
+    pre_collapse_counter = Counter()
+    replacement_samples = []
+    sample_count_by_group = {'beps': 0, 'frames': 0, 'mdx': 0}
+    corpus_freq = Counter()
+    paren_counter = Counter()
+    paren_samples = []
+
+    output = {
+        'meta': {
+            'pipeline_step': 4.2,
+            'description': 'PRE_COLLAPSE + pre-Kiwi substitution + user_dict. phrase_variants 기준.',
+            'user_dict': {'SL': USER_DICT_SL, 'NNG': USER_DICT_NNG},
+            'phrase_variants_count': len(phrase_variants),
+            'substitution_rule_count': len(subs),
+            'pre_collapse_rule_count': len(PRE_COLLAPSE),
+            'sample_quota': SAMPLE_QUOTA,
+            'notes': [
+                'DX canonical 유지 (디지털전환은 DX variant 로 통합)',
+                '3D모델/2D도면 canonical 은 Step 4 에서 제외 (3D/2D 일반 차원 토큰 보존)',
+                'PRE_COLLAPSE 는 regex 로 "디지털 전환(DX)" / "DX(DX)" 계열 통합',
+            ],
+        },
+        'beps': {}, 'frames': {}, 'mdx': {}, 'corpus': {},
+    }
+
+    beps = data.get('beps') or {}
+    if beps.get('text_nodes'):
+        key = f"beps:{beps['frame_id']}"
+        entry = process_source(kiwi, subs, 'beps', key, beps['text_nodes'],
+                               replacement_counter, pre_collapse_counter,
+                               replacement_samples, sample_count_by_group, corpus_freq,
+                               paren_counter, paren_samples)
+        output['beps'] = {'frame_id': beps['frame_id'], **entry}
+
+    for fid, info in data.get('frames', {}).items():
+        key = f"frame:{fid}"
+        entry = process_source(kiwi, subs, 'frames', key, info.get('text_nodes', []),
+                               replacement_counter, pre_collapse_counter,
+                               replacement_samples, sample_count_by_group, corpus_freq,
+                               paren_counter, paren_samples)
+        output['frames'][fid] = entry
+
+    for n, info in data.get('mdx', {}).items():
+        key = f"mdx:{n}"
+        entry = process_source(kiwi, subs, 'mdx', key, info.get('text_nodes', []),
+                               replacement_counter, pre_collapse_counter,
+                               replacement_samples, sample_count_by_group, corpus_freq,
+                               paren_counter, paren_samples)
+        output['mdx'][n] = entry
+
+    # corpus 통계
+    total_frames = len(output['frames'])
+    frame_df = Counter()
+    mdx_df = Counter()
+    for fid, info in output['frames'].items():
+        for tk in info['unique_tokens']:
+            frame_df[tk] += 1
+    for n, info in output['mdx'].items():
+        for tk in info['unique_tokens']:
+            mdx_df[tk] += 1
+
+    top50_by_freq = [
+        {'token': t, 'count': c,
+         'frame_count': frame_df.get(t, 0), 'mdx_count': mdx_df.get(t, 0)}
+        for t, c in corpus_freq.most_common(50)
+    ]
+    top50_by_frame_df = sorted(
+        [{'token': t, 'frame_count': frame_df[t],
+          'count': corpus_freq[t], 'mdx_count': mdx_df.get(t, 0)}
+         for t in frame_df],
+        key=lambda x: (-x['frame_count'], -x['count'], x['token']),
+    )[:50]
+
+    output['corpus'] = {
+        'unique_token_count': len(corpus_freq),
+        'total_occurrences': sum(corpus_freq.values()),
+        'top_50_by_frequency': top50_by_freq,
+        'top_50_by_frame_df': top50_by_frame_df,
+        'token_frequency': dict(sorted(corpus_freq.items(), key=lambda x: (-x[1], x[0]))),
+        'token_frame_df':  dict(sorted(frame_df.items(),    key=lambda x: (-x[1], x[0]))),
+        'token_mdx_df':    dict(sorted(mdx_df.items(),      key=lambda x: (-x[1], x[0]))),
+    }
+
+    output['meta']['totals'] = {
+        'beps_raw': output['beps'].get('raw_token_count', 0),
+        'beps_unique': output['beps'].get('unique_token_count', 0),
+        'frames_raw': sum(f['raw_token_count'] for f in output['frames'].values()),
+        'frames_unique_avg': (
+            sum(f['unique_token_count'] for f in output['frames'].values()) / total_frames
+            if total_frames else 0
+        ),
+        'mdx_raw': sum(m['raw_token_count'] for m in output['mdx'].values()),
+        'mdx_unique_avg': (
+            sum(m['unique_token_count'] for m in output['mdx'].values()) / len(output['mdx'])
+            if output['mdx'] else 0
+        ),
+        'corpus_unique': len(corpus_freq),
+    }
+
+    OUTPUT_TOKENS.write_text(
+        yaml.safe_dump(output, allow_unicode=True, sort_keys=False, width=200),
+        encoding='utf-8',
+    )
+
+    pre_collapse_total = sum(pre_collapse_counter.values())
+    phrase_total = sum(replacement_counter.values())
+    total_text_changes = pre_collapse_total + phrase_total
+
+    report = {
+        'meta': {
+            'pipeline_step': 4.2,
+            'description': 'PRE_COLLAPSE + 괄호 확장 + phrase_variants 치환 카운트 + per-group 쿼터 샘플',
+            'pre_collapse_rule_count': len(PRE_COLLAPSE),
+            'substitution_rule_count': len(subs),
+            'sample_quota': SAMPLE_QUOTA,
+            'paren_expansion_max_len': MAX_PAREN_LEN,
+        },
+        'pre_collapse_applied': dict(
+            sorted(pre_collapse_counter.items(), key=lambda x: -x[1])
+        ),
+        'pre_collapse_total': pre_collapse_total,
+        'paren_expanded': {
+            'total': paren_counter.get('expansions', 0),
+            'samples_count': len(paren_samples),
+            'samples': paren_samples,
+        },
+        'applied_replacements': dict(
+            sorted(replacement_counter.items(), key=lambda x: -x[1])
+        ),
+        'total_replacements': phrase_total,
+        'total_text_changes': total_text_changes,
+        'sample_count_by_group': sample_count_by_group,
+        'samples': replacement_samples,
+    }
+    OUTPUT_REPORT.write_text(
+        yaml.safe_dump(report, allow_unicode=True, sort_keys=False, width=200),
+        encoding='utf-8',
+    )
+
+    # 콘솔 요약
+    t = output['meta']['totals']
+    print(f"[Step 4.2] PRE_COLLAPSE (4 rules) + pre-Kiwi substitution + user_dict")
+    print(f"  PRE_COLLAPSE: {len(PRE_COLLAPSE)} regex rules")
+    print(f"  phrase_variants: {len(phrase_variants)}개 canonical, {len(subs)}개 rule")
+    print(f"  user_dict: SL {len(USER_DICT_SL)}, NNG {len(USER_DICT_NNG)}")
+    print()
+    print(f"  BEPS:   raw={t['beps_raw']}, unique={t['beps_unique']}")
+    print(f"  Frames: raw={t['frames_raw']}, unique_avg={t['frames_unique_avg']:.1f} ({total_frames}개 frame)")
+    print(f"  MDX:    raw={t['mdx_raw']}, unique_avg={t['mdx_unique_avg']:.1f}")
+    print(f"  Corpus: unique={t['corpus_unique']}, occurrences={sum(corpus_freq.values())}")
+    print()
+    print(f"  pre_collapse_total:    {pre_collapse_total}")
+    for rid, cnt in report['pre_collapse_applied'].items():
+        print(f"    [{rid}] {cnt}")
+    print(f"  applied_replacements total: {phrase_total}")
+    for canonical, cnt in report['applied_replacements'].items():
+        print(f"    {canonical:10s} {cnt}")
+    print(f"  total_text_changes:    {total_text_changes}")
+    print()
+    print(f"  samples per group: {sample_count_by_group} (quota={SAMPLE_QUOTA})")
+    print()
+    print(f"산출: {OUTPUT_TOKENS}")
+    print(f"      {OUTPUT_REPORT}")
+
+
+if __name__ == "__main__":
+    main()