85 lines
3.0 KiB
Python
85 lines
3.0 KiB
Python
"""IDF-only 매칭: TF 빼고 공통 희귀 단어의 IDF 합으로만 매칭"""
|
|
import sys
|
|
import json
|
|
import math
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
from common import load_figma_texts, tokenize_simple
|
|
from extract_units import extract_units
|
|
|
|
|
|
def main():
|
|
figma = load_figma_texts()
|
|
units = extract_units()
|
|
|
|
# 전체 corpus IDF
|
|
all_docs = {}
|
|
for uid, text in units.items():
|
|
all_docs[f"MDX:{uid}"] = tokenize_simple(text)
|
|
for fid, text in figma.items():
|
|
all_docs[f"FIG:{fid}"] = tokenize_simple(text)
|
|
df = Counter()
|
|
for toks in all_docs.values():
|
|
for w in set(toks):
|
|
df[w] += 1
|
|
N = len(all_docs)
|
|
idf = {w: math.log(N / c) for w, c in df.items()}
|
|
|
|
with open(Path(__file__).parent.parent.parent / "data" / "figma_previews" / "index.json",
|
|
encoding="utf-8") as f:
|
|
idx_data = json.load(f)
|
|
frame_to_short = {info["frame_id"]: sid for sid, info in idx_data.items()}
|
|
|
|
def idf_sum_score(query_tokens, doc_tokens, idf_dict):
|
|
"""공통 단어의 IDF 합 (단순)"""
|
|
common = set(query_tokens) & set(doc_tokens)
|
|
return sum(idf_dict.get(w, 0) for w in common)
|
|
|
|
def idf_jaccard(query_tokens, doc_tokens, idf_dict):
|
|
"""IDF 가중 Jaccard = IDF(교집합) / IDF(합집합)"""
|
|
q_set = set(query_tokens)
|
|
d_set = set(doc_tokens)
|
|
common = q_set & d_set
|
|
union = q_set | d_set
|
|
num = sum(idf_dict.get(w, 0) for w in common)
|
|
den = sum(idf_dict.get(w, 0) for w in union)
|
|
return num / den if den > 0 else 0
|
|
|
|
# 핵심: MDX03-1 테스트
|
|
test_units = ["MDX03-1", "MDX03-2", "MDX01-2", "MDX02-1", "MDX02-2", "MDX01-1"]
|
|
|
|
for uid in test_units:
|
|
if uid not in units:
|
|
continue
|
|
print(f"\n═══════════════════════════════════════")
|
|
print(f"{uid}")
|
|
print(f"═══════════════════════════════════════")
|
|
query_toks = tokenize_simple(units[uid])
|
|
|
|
# IDF sum
|
|
scores_sum = []
|
|
scores_jac = []
|
|
for fid, text in figma.items():
|
|
doc_toks = tokenize_simple(text)
|
|
scores_sum.append((fid, idf_sum_score(query_toks, doc_toks, idf)))
|
|
scores_jac.append((fid, idf_jaccard(query_toks, doc_toks, idf)))
|
|
scores_sum.sort(key=lambda x: -x[1])
|
|
scores_jac.sort(key=lambda x: -x[1])
|
|
|
|
print(f"\n[IDF-Sum] (공통 희귀 단어 IDF 합)")
|
|
for i, (fid, sc) in enumerate(scores_sum[:5], 1):
|
|
short = frame_to_short.get(str(fid), "?")
|
|
print(f" {i}. {short} score={sc:.2f}")
|
|
|
|
print(f"\n[IDF-Jaccard] (IDF 가중 Jaccard)")
|
|
for i, (fid, sc) in enumerate(scores_jac[:5], 1):
|
|
short = frame_to_short.get(str(fid), "?")
|
|
print(f" {i}. {short} score={sc:.3f}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|