"""Template-fit-v1 scoring engine. Spec: tests/matching/TEMPLATE_FIT_V1.md 공식: base = 0.25*anchor + 0.20*card + 0.20*relation + 0.15*slot + 0.20*content total_penalty = min(0.50, min(0.30, adaptation_cost) + min(0.30, not_suits_hits*0.20)) confidence = max(0, base - total_penalty) v1 특성: - 매칭 단계 LLM 호출 없음 (코드 only) - content_embedding은 mock dict (ko-sroberta 연결은 검증 통과 후) - MDX analysis는 4 TARGET 수동 fixture (detect_mdx_analysis() 구현은 나중) """ import yaml from pathlib import Path from detect_mdx import detect_mdx_analysis from phase_common import TARGET_UNITS, load_target_units from embeddings import embed_texts, cosine HERE = Path(__file__).parent # ─── 가중치 ──────────────────────────────────────────── W_ANCHOR = 0.25 W_CARDINALITY = 0.20 W_RELATION = 0.20 W_SLOT = 0.15 W_CONTENT = 0.20 # ─── 캡 ────────────────────────────────────────────── CAP_ADAPTATION = 0.30 CAP_NOT_SUITS = 0.30 CAP_TOTAL = 0.50 NOT_SUITS_HIT_PENALTY = 0.20 # ─── adaptation 조작별 비용 ────────────────────────── ADAPT_COST = { 'split': 0.10, 'merge': 0.15, 'infer_missing_slot': 0.25, 'rewrite_label': 0.05, 'rewrite_body': 0.05, } # ─── structure_intent 호환 매트릭스 ─────────────────── # 0~1 범위. 높을수록 "이 콘텐츠를 이 구조로 옮기기 쉬움". # Symmetric 기본. 1.0 = 완전 일치, 0.2 이하 = 거의 불가능. INTENT_COMPAT = { # exact match ('concept_comparison', 'concept_comparison'): 1.0, ('multi_attribute_comparison', 'multi_attribute_comparison'): 1.0, ('transformation_story', 'transformation_story'): 1.0, ('process_product_split', 'process_product_split'): 1.0, ('category_comparison', 'category_comparison'): 1.0, ('industry_comparison', 'industry_comparison'): 1.0, ('persona_benefit', 'persona_benefit'): 1.0, ('requirement_list', 'requirement_list'): 1.0, ('problem_diagnosis', 'problem_diagnosis'): 1.0, ('requirement_or_pillar', 'requirement_or_pillar'): 1.0, # strong compat (same family) ('concept_comparison', 'multi_attribute_comparison'): 0.8, ('multi_attribute_comparison', 'category_comparison'): 0.8, ('multi_attribute_comparison', 'industry_comparison'): 0.8, ('transformation_story', 'process_product_split'): 0.9, ('requirement_list', 'requirement_or_pillar'): 0.7, # moderate (partial overlap) ('concept_comparison', 'category_comparison'): 0.5, ('multi_attribute_comparison', 'persona_benefit'): 0.4, ('industry_comparison', 'concept_comparison'): 0.4, ('requirement_or_pillar', 'persona_benefit'): 0.4, # weak (different family) ('concept_comparison', 'transformation_story'): 0.2, ('concept_comparison', 'process_product_split'): 0.2, ('multi_attribute_comparison', 'transformation_story'): 0.2, ('multi_attribute_comparison', 'process_product_split'): 0.2, ('category_comparison', 'transformation_story'): 0.2, ('industry_comparison', 'transformation_story'): 0.2, # polarity opposite ('requirement_list', 'problem_diagnosis'): 0.15, ('requirement_or_pillar', 'problem_diagnosis'): 0.25, ('persona_benefit', 'problem_diagnosis'): 0.2, # structurally different (compare vs parallel world) ('persona_benefit', 'concept_comparison'): 0.25, ('requirement_list', 'concept_comparison'): 0.2, ('requirement_list', 'multi_attribute_comparison'): 0.3, ('requirement_list', 'persona_benefit'): 0.3, ('requirement_list', 'category_comparison'): 0.2, ('requirement_list', 'industry_comparison'): 0.25, ('requirement_list', 'process_product_split'): 0.2, ('requirement_list', 'transformation_story'): 0.2, ('problem_diagnosis', 'concept_comparison'): 0.2, ('problem_diagnosis', 'multi_attribute_comparison'): 0.25, ('problem_diagnosis', 'category_comparison'): 0.2, ('problem_diagnosis', 'industry_comparison'): 0.2, ('problem_diagnosis', 'process_product_split'): 0.15, ('problem_diagnosis', 'transformation_story'): 0.15, } def _sym_lookup(a, b, default=0.3): """대칭 조회. (a,b) 또는 (b,a) 있으면 반환, 아니면 default.""" if (a, b) in INTENT_COMPAT: return INTENT_COMPAT[(a, b)] if (b, a) in INTENT_COMPAT: return INTENT_COMPAT[(b, a)] return default def intent_compat(mdx_intents, frame_intents): """MDX vs Frame 의 structure_intent list 호환도 계산. - 둘 다 non-empty: max over all pairs - 한쪽이 empty: neutral 0.5 (부분 적용 상태 — intent 미태깅 frame 배려) """ if not mdx_intents or not frame_intents: return 0.5 return max(_sym_lookup(mi, fi) for mi in mdx_intents for fi in frame_intents) def intent_compat_with_source(mdx_intents, frame_intents): """호환도 + source ('tagged' | 'neutral'). Phase 25 gate 에서 tagged mismatch 만 강하게 막기 위해 사용.""" if not mdx_intents or not frame_intents: return 0.5, 'neutral' return (max(_sym_lookup(mi, fi) for mi in mdx_intents for fi in frame_intents), 'tagged') # ─── relation 호환 매트릭스 ────────────────────────── RELATION_MATRIX = { ('parallel', 'parallel'): 1.0, ('parallel', 'sequence'): 0.2, ('parallel', 'compare'): 0.4, ('parallel', 'hierarchy'): 0.3, ('sequence', 'parallel'): 0.2, ('sequence', 'sequence'): 1.0, ('sequence', 'compare'): 0.3, ('sequence', 'hierarchy'): 0.4, ('compare', 'parallel'): 0.4, ('compare', 'sequence'): 0.3, ('compare', 'compare'): 1.0, ('compare', 'hierarchy'): 0.3, ('hierarchy', 'parallel'): 0.3, ('hierarchy', 'sequence'): 0.4, ('hierarchy', 'compare'): 0.3, ('hierarchy', 'hierarchy'): 1.0, } # ─── not_suits 규칙 엔진 (pattern 키워드 → MDX 시그널 판정) ── # v1: structured signal 기반. 나중에 fit_notes.not_suits를 structured로 확장 예정. NOT_SUITS_RULES = { '시간 순서': lambda m: m['relation_type']['value'] == 'sequence', '시간 순서 단계': lambda m: m['relation_type']['value'] == 'sequence', '원인-결과': lambda m: m['relation_type']['value'] == 'sequence', '2개 비교': lambda m: m['relation_type']['value'] == 'compare' and m['item_count']['detected'] == 2, '2주체만': lambda m: m['item_count']['detected'] == 2, '3개 이상 비교': lambda m: m['relation_type']['value'] == 'compare' and m['item_count']['detected'] >= 3, '3개 이상 병렬': lambda m: m['relation_type']['value'] == 'parallel' and m['item_count']['detected'] >= 3, '4개 이상': lambda m: m['item_count']['detected'] >= 4, '4주체 이상': lambda m: m['item_count']['detected'] >= 4, '1개 개념': lambda m: m['item_count']['detected'] == 1, '단일 주체': lambda m: m['item_count']['detected'] == 1, '단일 주제': lambda m: m['item_count']['detected'] == 1, '단일 산업': lambda m: m['item_count']['detected'] == 1, '단일 관점': lambda m: m['item_count']['detected'] == 1, '병렬 나열': lambda m: m['relation_type']['value'] == 'parallel', '주체별 나열': lambda m: any(p in m.get('detected_terms', []) for p in ['발주자', '시공자', '설계자', '주체별']), '산업 비교': lambda m: any(p in m.get('detected_terms', []) for p in ['제조업', '건축', '토목', '산업별']), '필수요건': lambda m: any(p in m.get('detected_terms', []) for p in ['필수조건', '필수요건', '필수']), '요건 나열': lambda m: any(p in m.get('detected_terms', []) for p in ['필수조건', '필수요건']), 'BIM vs DX 직접 대조': lambda m: ('BIM' in m.get('detected_terms', []) and 'DX' in m.get('detected_terms', []) and m['relation_type']['value'] == 'compare' and m['item_count']['detected'] == 2), } def load_templates_v1(): path = HERE / 'structure_ontology.yaml' with open(path, encoding='utf-8') as f: data = yaml.safe_load(f) assert data['meta']['schema_version'] == 'template-fit-v1', \ f"schema_version mismatch: {data['meta']['schema_version']}" return data['templates_v1'] # ═══ 각 축별 점수 계산 ═══ def anchor_match(detected_terms, anchor_sets): """max across anchor_sets with per-set conditional cap. per-set options (anchor_sets[i]): min_hits → 이 수치 미만이면 해당 set 무시 (기본 1) confidence_cap → ratio 상한 (기본 1.0 = 미적용) cap_exempt_if_corroborated_by → 같은 템플릿의 다른 set 최고 ratio가 이 값 이상이면 cap 면제 (기본 None) returns: (effective_ratio, set_id, matched_terms, cap_note) """ if not anchor_sets: return 0.0, None, set(), '' detected_set = set(detected_terms) # 1. min_hits 통과하는 set 만 수집 raw = [] for s in anchor_sets: terms = s['terms'] min_hits = s.get('min_hits', 1) matched = detected_set & set(terms) if len(matched) < min_hits: continue raw.append({ 'id': s['id'], 'raw_ratio': len(matched) / len(terms) if terms else 0.0, 'matched': matched, 'cap': s.get('confidence_cap', 1.0), 'exempt': s.get('cap_exempt_if_corroborated_by', None), }) if not raw: return 0.0, None, set(), '' # 2. 각 set 별 effective ratio 계산 (cap 조건부 적용) for r in raw: others_max = max( (x['raw_ratio'] for x in raw if x['id'] != r['id']), default=0.0, ) if r['cap'] < 1.0 and r['raw_ratio'] > r['cap']: if r['exempt'] is not None and others_max >= r['exempt']: r['effective'] = r['raw_ratio'] r['note'] = f"cap{r['cap']:.2f} 면제 (방증 {others_max:.2f}≥{r['exempt']})" else: r['effective'] = r['cap'] r['note'] = f"cap{r['cap']:.2f} 적용 (방증 {others_max:.2f}<{r['exempt']})" else: r['effective'] = r['raw_ratio'] r['note'] = '' # 3. effective ratio 최대 set 선택 best = max(raw, key=lambda x: x['effective']) return best['effective'], best['id'], best['matched'], best['note'] def cardinality_match(mdx_count, card, adaptation_allowed): ideal, mn, mx = card['ideal'], card['min'], card['max'] if mdx_count == ideal: return 1.0 if mn <= mdx_count <= mx: return 0.8 if adaptation_allowed.get('split') or adaptation_allowed.get('merge'): return 0.5 return 0.0 def relation_match(mdx_rel, frame_rel): if mdx_rel == frame_rel: return 1.0 return RELATION_MATRIX.get((mdx_rel, frame_rel), 0.2) def slot_coverage(mdx_analysis, template): """rough v1: 0.5 (cardinality 범위 내) + 0.3 (라벨) + 0.2 (본문).""" n = mdx_analysis['item_count']['detected'] card = template['visual_pattern']['cardinality'] within = card['min'] <= n <= card['max'] candidates = mdx_analysis.get('slot_candidates', []) has_labels = bool(candidates) and all(sc.get('label') for sc in candidates) has_bodies = bool(candidates) and all(sc.get('body') for sc in candidates) score = 0.0 if within: score += 0.5 if has_labels: score += 0.3 if has_bodies: score += 0.2 return score def adaptation_cost(mdx_analysis, template): """Cardinality mismatch 시만 비용 부과. v1에서는 rewrite cost 안 부과 (light_edit 단계 책임).""" card = template['visual_pattern']['cardinality'] allowed = template['adaptation_allowed'] n = mdx_analysis['item_count']['detected'] ops = [] cost = 0.0 if n < card['min']: if allowed.get('split'): ops.append(f'split({n}→{card["min"]})') cost += ADAPT_COST['split'] elif allowed.get('infer_missing_slot'): ops.append('infer_missing') cost += ADAPT_COST['infer_missing_slot'] else: ops.append('forbidden:n card['max']: if allowed.get('merge'): ops.append(f'merge({n}→{card["max"]})') cost += ADAPT_COST['merge'] else: ops.append('forbidden:n>max') return min(CAP_ADAPTATION, cost), ops def not_suits_penalty(mdx_analysis, template): not_suits = template.get('fit_notes', {}).get('not_suits', []) hits = 0 matched = [] for pattern in not_suits: fired = False # 가장 구체적인 키 먼저 매치 for key, rule in sorted(NOT_SUITS_RULES.items(), key=lambda x: -len(x[0])): if key in pattern: if rule(mdx_analysis): fired = True break if fired: hits += 1 matched.append(pattern) penalty = min(CAP_NOT_SUITS, hits * NOT_SUITS_HIT_PENALTY) return penalty, matched # ═══ 최종 confidence 계산 ═══ def compute_template_fit(mdx_analysis, template, content_embedding): anchor_s, anchor_set_id, anchor_terms, anchor_note = anchor_match( mdx_analysis['detected_terms'], template['anchor_sets'] ) card_s = cardinality_match( mdx_analysis['item_count']['detected'], template['visual_pattern']['cardinality'], template['adaptation_allowed'], ) rel_s = relation_match( mdx_analysis['relation_type']['value'], template['visual_pattern']['relation_type'], ) slot_s = slot_coverage(mdx_analysis, template) content_s = content_embedding # structure_intent 호환도 (score 에 반영하지 않고 gate 용으로만 저장) mdx_intents = mdx_analysis.get('structure_intent', []) frame_intents = template.get('visual_pattern', {}).get('structure_intent', []) ic, ic_source = intent_compat_with_source(mdx_intents, frame_intents) base = (W_ANCHOR * anchor_s + W_CARDINALITY * card_s + W_RELATION * rel_s + W_SLOT * slot_s + W_CONTENT * content_s) adapt_pen, adapt_ops = adaptation_cost(mdx_analysis, template) ns_pen, ns_matched = not_suits_penalty(mdx_analysis, template) total_pen = min(CAP_TOTAL, adapt_pen + ns_pen) confidence = max(0.0, base - total_pen) return { 'confidence': confidence, 'base': base, 'total_penalty': total_pen, 'axes': { 'anchor': {'score': anchor_s, 'set_id': anchor_set_id, 'terms': sorted(anchor_terms), 'note': anchor_note}, 'cardinality': card_s, 'relation': rel_s, 'slot': slot_s, 'content': content_s, 'intent': {'compat': ic, 'source': ic_source, 'mdx': mdx_intents, 'frame': frame_intents}, }, 'adaptation': {'penalty': adapt_pen, 'ops': adapt_ops}, 'not_suits': {'penalty': ns_pen, 'matched': ns_matched}, } def route(confidence, axes=None, adaptation=None, not_suits=None): """Multi-gate routing. 1. Intent gate (tagged mismatch 만 강하게 차단) 2. Route_v2 multi-constraint (anchor/content/adaptation/not_suits) axes/adaptation/not_suits 미제공 시 legacy confidence-only 로 fallback. """ # Legacy fallback (구 호출자 호환) if axes is None: if confidence >= 0.90: return 'use_as_is' if confidence >= 0.75: return 'light_edit' if confidence >= 0.60: return 'restructure' return 'reject' a = axes['anchor']['score'] c = axes['content'] adapt_pen = adaptation['penalty'] if adaptation else 0.0 ns_hits = len(not_suits['matched']) if not_suits else 0 intent = axes.get('intent', {}) ic = intent.get('compat', 0.5) ic_source = intent.get('source', 'neutral') # ─── Intent gate (tagged mismatch 만 강하게) ──────── if ic_source == 'tagged': if ic < 0.4: return 'reject' if ic < 0.7: # use_as_is/light_edit 금지, restructure 이하만 가능 if (confidence >= 0.60 and (a >= 0.35 or (c >= 0.55 and adapt_pen < 0.20)) and ns_hits <= 1): return 'restructure' return 'reject' # ─── Route_v2 multi-constraint (normal path) ────── # use_as_is: 엄격 if (confidence >= 0.90 and a >= 0.70 and c >= 0.55 and ns_hits == 0): return 'use_as_is' # light_edit: 중간 if (confidence >= 0.75 and a >= 0.50 and c >= 0.45 and adapt_pen < 0.10): return 'light_edit' # restructure: 최소 증거 if (confidence >= 0.60 and (a >= 0.35 or (c >= 0.55 and adapt_pen < 0.20)) and ns_hits <= 1): return 'restructure' return 'reject' # ═══ anchor_vocab 수집 ═══ def collect_anchor_vocab(templates): vocab = set() for tpl in templates.values(): for s in tpl['anchor_sets']: vocab.update(s['terms']) return vocab # ═══ 4 TARGET MDX analysis ═══ # v1 fixture 방식 (참고용, 실제 실행은 load_mdx_analyses 사용) _LEGACY_FIXTURE = { 'MDX01-2-details': { 'title': 'BIM과 DX의 이해', 'summary': 'BIM과 DX 용어 혼용 정리. 범위·S/W·프로세스·성과품·활용·확장성·수행개념·주체 관점별 비교.', 'detected_terms': ['BIM', 'DX', '범위', '성과품', '확장성', '수행개념', '수행주체', '프로세스', '활용', '비교', '관점별', '용어비교', '상호관계', '혼용'], 'item_count': {'detected': 2, 'source': 'table_columns'}, 'relation_type': {'value': 'compare', 'confidence': 'high'}, 'content_shape': {'has_table': True, 'has_subsections': False, 'has_bullets': False}, 'slot_candidates': [ {'label': 'BIM', 'body': 'Building Information Modeling (범위/S·W/프로세스 ...)'}, {'label': 'DX', 'body': 'Digital Transformation (범위/S·W/프로세스 ...)'}, ], }, 'MDX02-2.2-table': { 'title': '주체별 기대효과', 'summary': 'DX 시행 주체별 기대효과. 발주자·시공자·설계자 각각의 역할별 목표와 기대 효과.', 'detected_terms': ['발주자', '시공자', '설계자', '주체별', '기대효과', '역할', '역량목표', '소통', '협업', '품질향상', '생산성향상'], 'item_count': {'detected': 3, 'source': 'table_rows'}, 'relation_type': {'value': 'parallel', 'confidence': 'high'}, 'content_shape': {'has_table': True, 'has_subsections': False, 'has_bullets': True}, 'slot_candidates': [ {'label': '발주자', 'body': '소통·행정자동화·공사Risk 최소화'}, {'label': '시공자', 'body': '품질향상·생산성향상·현장실효성'}, {'label': '설계자', 'body': '오류예방·리스크최소화·민원예방'}, ], }, 'MDX03-1': { 'title': 'DX 시행을 위한 필수요건', 'summary': 'DX 시행을 위한 3대 필수조건. 기술·사람·자연 여건의 조화.', 'detected_terms': ['DX', '기술', '사람', '자연', '필수조건', '역량', '여건', '디지털기술', '3요소'], 'item_count': {'detected': 3, 'source': 'bullets'}, 'relation_type': {'value': 'parallel', 'confidence': 'high'}, 'content_shape': {'has_table': False, 'has_subsections': False, 'has_bullets': True}, 'slot_candidates': [ {'label': '기술', 'body': '디지털기술과 기반지식'}, {'label': '사람', 'body': '역량과 창의성'}, {'label': '자연', 'body': '여건과 투자 기반'}, ], }, 'MDX03-2': { 'title': 'Process/Product 혁신', 'summary': 'Process 혁신과 Product 혁신. 과정의 Analogue→Digital Transformation, 결과의 2D도면→3D모델 전환.', 'detected_terms': ['과정혁신', '결과혁신', 'Process', 'Product', 'Analogue', 'Digital', '2D', '3D', 'Transformation', 'AS-IS', 'TO-BE', '2D도면', '3D모델'], 'item_count': {'detected': 2, 'source': 'subsections'}, 'relation_type': {'value': 'compare', 'confidence': 'high'}, 'content_shape': {'has_table': False, 'has_subsections': True, 'has_bullets': True}, 'slot_candidates': [ {'label': '과정혁신', 'body': 'Analogue → Digital Transformation'}, {'label': '결과혁신', 'body': '2D도면 → 3D모델'}, ], }, } # ═══ Mock content embedding ═══ # 정답 페어 0.90, 관련 0.45~0.55, 무관 0.40 CONTENT_EMBEDDING_MOCK = { 'MDX01-2-details': { 'bim_dx_comparison_table': 0.90, 'process_product_two_way': 0.55, 'sw_reality_three_emphasis': 0.50, 'three_persona_benefits': 0.45, 'three_parallel_requirements':0.40, }, 'MDX02-2.2-table': { 'three_persona_benefits': 0.90, 'three_parallel_requirements':0.50, 'sw_reality_three_emphasis': 0.45, 'bim_dx_comparison_table': 0.40, 'process_product_two_way': 0.40, }, 'MDX03-1': { 'three_parallel_requirements':0.90, 'sw_reality_three_emphasis': 0.55, 'three_persona_benefits': 0.50, 'process_product_two_way': 0.50, 'bim_dx_comparison_table': 0.45, }, 'MDX03-2': { 'process_product_two_way': 0.90, 'bim_dx_comparison_table': 0.55, 'sw_reality_three_emphasis': 0.50, 'three_parallel_requirements':0.45, 'three_persona_benefits': 0.40, }, } GROUND_TRUTH = { 'MDX01-2-details': 'bim_dx_comparison_table', 'MDX02-2.2-table': 'three_persona_benefits', 'MDX03-1': 'three_parallel_requirements', 'MDX03-2': 'process_product_two_way', } def load_mdx_analyses(anchor_vocab): """실제 MDX 소스에서 detect_mdx_analysis() 로 4 TARGET 분석 생성.""" units_full, units_title = load_target_units() out = {} for t in TARGET_UNITS: uid = t[0] out[uid] = detect_mdx_analysis(units_full[uid], units_title[uid], anchor_vocab) return out def compute_content_sim(mdx_analyses, tpl_by_id): """ko-sroberta cosine: MDX summary ↔ template description.""" print(f" [embedding] ko-sroberta load + {len(tpl_by_id)} template + {len(mdx_analyses)} MDX encode ...") tpl_descs = {tid: tpl['description'] for tid, tpl in tpl_by_id.items()} tpl_ids = list(tpl_descs.keys()) tpl_vecs = embed_texts([tpl_descs[i] for i in tpl_ids]) mdx_ids = list(mdx_analyses.keys()) mdx_summaries = [mdx_analyses[i]['summary'] for i in mdx_ids] mdx_vecs = embed_texts(mdx_summaries) sim = {} for i, mid in enumerate(mdx_ids): sim[mid] = {} for j, tid in enumerate(tpl_ids): sim[mid][tid] = max(0.0, min(1.0, cosine(mdx_vecs[i], tpl_vecs[j]))) return sim # ─── 검증 기준 ───────────────────────────────────────── # 리뷰어 합의 기준: # 1. 정답 후보는 1위여야 한다 # 2. 정답 아닌 후보는 use_as_is(≥0.90) 라우팅되면 안 된다 # 3. 1위와 2위의 격차는 최소 0.05 이상 (fragile tie 방지) MIN_GAP = 0.05 USE_AS_IS_FLOOR = 0.90 def main(): templates = load_templates_v1() tpl_by_id = {t['template_id']: t for t in templates.values()} vocab = collect_anchor_vocab(templates) print(f"=== Template-fit v1 scoring — ko-sroberta content embedding ===\n") MDX_ANALYSES = load_mdx_analyses(vocab) CONTENT_SIM = compute_content_sim(MDX_ANALYSES, tpl_by_id) print() hits = 0 warnings = [] for mdx_id, mdx_analysis in MDX_ANALYSES.items(): gt = GROUND_TRUTH[mdx_id] print(f"━━ {mdx_id} 정답: {gt}") print(f" item_count={mdx_analysis['item_count']['detected']:d} " f"relation={mdx_analysis['relation_type']['value']} " f"terms={len(mdx_analysis['detected_terms'])}") results = [] for tpl_id, tpl in tpl_by_id.items(): content_sim = CONTENT_SIM[mdx_id][tpl_id] r = compute_template_fit(mdx_analysis, tpl, content_sim) results.append((tpl_id, r)) results.sort(key=lambda x: -x[1]['confidence']) for rank, (tpl_id, r) in enumerate(results, 1): a = r['axes'] mark = '✓' if tpl_id == gt else ' ' route_label = route(r['confidence']) a_score = a['anchor']['score'] a_set = a['anchor']['set_id'] or '-' a_terms = a['anchor']['terms'] a_note = a['anchor'].get('note', '') print(f" [{rank}] {mark} {tpl_id:32s} conf={r['confidence']:.3f} ({route_label})") suffix = f" {a_note}" if a_note else '' print(f" anchor={a_score:.2f}({a_set}: {a_terms}){suffix}") print(f" card={a['cardinality']:.2f} rel={a['relation']:.2f} slot={a['slot']:.2f} content={a['content']:.2f}") print(f" base={r['base']:.3f} " f"adapt=-{r['adaptation']['penalty']:.2f}{r['adaptation']['ops']} " f"nsuits=-{r['not_suits']['penalty']:.2f}{r['not_suits']['matched']}") # 검증 체크 top1_tpl, top1_r = results[0] top2_tpl, top2_r = results[1] if top1_tpl == gt: hits += 1 else: warnings.append(f"[{mdx_id}] top1={top1_tpl} ≠ gt={gt}") # 정답 아닌 후보가 use_as_is 인지 for tpl_id, r in results: if tpl_id != gt and r['confidence'] >= USE_AS_IS_FLOOR: warnings.append( f"[{mdx_id}] 오답 후보 use_as_is: {tpl_id} conf={r['confidence']:.3f}" ) # 1-2위 격차 gap = top1_r['confidence'] - top2_r['confidence'] if gap < MIN_GAP: warnings.append( f"[{mdx_id}] 격차 {gap:.3f} < 기준 {MIN_GAP} " f"(top1={top1_tpl} {top1_r['confidence']:.3f} / " f"top2={top2_tpl} {top2_r['confidence']:.3f})" ) print() print(f"=== Result: {hits}/{len(MDX_ANALYSES)} ===\n") if warnings: print("⚠️ 경고") for w in warnings: print(f" - {w}") else: print(f"✓ 검증 기준 통과 ({hits}/{len(MDX_ANALYSES)}, " f"오답 use_as_is 없음, 1-2위 격차 ≥ {MIN_GAP})") if __name__ == '__main__': main()