From 886192ae58dd5c2094fa72b33d946510ee0d68a1 Mon Sep 17 00:00:00 2001 From: b17301 Date: Mon, 8 Jun 2026 14:25:47 +0900 Subject: [PATCH] Preserve latest comparison functionality --- main.py | 1386 ++++++- scripts/prune_wehago_projection_history.py | 120 + scripts/rebuild_compare_ranges.py | 15 +- scripts/reconcile_wehago_projection_to_db.py | 1712 ++++++++- scripts/run_wehago_logic_iteration.py | 259 ++ templates/admin_users.html | 213 +- templates/base.html | 28 +- templates/cost_analysis.html | 125 +- templates/login.html | 2 +- templates/wehago_benefit_entertainment.html | 921 +++++ templates/wehago_compare.html | 66 +- wehago_compare.py | 3604 ++++++++++++++++-- 12 files changed, 7966 insertions(+), 485 deletions(-) create mode 100644 scripts/prune_wehago_projection_history.py create mode 100644 scripts/run_wehago_logic_iteration.py create mode 100644 templates/wehago_benefit_entertainment.html diff --git a/main.py b/main.py index 477d7f5..30df8ac 100644 --- a/main.py +++ b/main.py @@ -17,11 +17,12 @@ import time import tempfile import uuid import zipfile -from datetime import date, datetime, timedelta +from io import BytesIO +from datetime import date, datetime, timedelta, timezone from decimal import Decimal, InvalidOperation, ROUND_HALF_UP from functools import lru_cache from pathlib import Path -from typing import Any +from typing import Any, Mapping, Sequence from urllib.parse import parse_qs, quote_plus, unquote_plus import uvicorn @@ -30,7 +31,9 @@ from fastapi.encoders import jsonable_encoder from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from openpyxl import load_workbook +from openpyxl import Workbook, load_workbook +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils import get_column_letter from sqlalchemy import bindparam, create_engine, event, text from sqlalchemy.engine import URL from sqlalchemy.exc import OperationalError @@ -54,6 +57,7 @@ from wehago_compare import ( _offset_group_vector, _offset_vectors_cancel_each_other, _voucher_groups_within_days, + apply_wehago_final_status_counts_to_metric_sections, cleanup_compare_runtime_artifacts, enqueue_default_pair_recommend_precompute, export_wehago_status_rows_xlsx, @@ -64,8 +68,10 @@ from wehago_compare import ( get_status_export_job, get_status_field_suggestions, get_status_detail_rows, + get_dashboard_metric_counts_nonblocking, get_wehago_compare_dashboard, get_wehago_compare_summary, + get_wehago_final_status_summary_from_conn, get_wehago_filtered_rows, import_uploaded_erp_voucher_file, init_wehago_compare_db, @@ -136,6 +142,7 @@ AUTH_PERMISSION_BY_PREFIX = { "/process-cost": "process_cost", "/cost-analysis": "cost_analysis", "/projects": "projects", + "/wehago-benefit-entertainment": "wehago_compare", "/wehago-compare": "wehago_compare", "/hanmac-browser": "hanmac_browser", "/db-browser": "db_browser", @@ -150,6 +157,7 @@ AUTH_NAV_ITEMS = [ {"href": "/cost-analysis", "label": "프로젝트 손익분석", "permission": "cost_analysis", "active": "exact"}, {"href": "/projects", "label": "프로젝트 정보", "permission": "projects", "active": "exact"}, {"href": "/wehago-compare", "label": "전표비교", "permission": "wehago_compare", "active": "exact"}, + {"href": "/wehago-benefit-entertainment", "label": "복리/접대비", "permission": "wehago_compare", "active": "exact"}, {"href": "/hanmac-browser", "label": "hanmac DB_external", "permission": "hanmac_browser", "active": "exact"}, {"href": "/db-browser", "label": "DB 조회", "permission": "db_browser", "active": "db"}, {"href": "/admin/users", "label": "사용자 관리", "permission": "admin", "active": "admin"}, @@ -186,7 +194,7 @@ PROJECT_ACCOUNT_BREAKDOWN_CACHE_TTL_SECONDS = 300.0 _COST_ANALYSIS_PAYLOAD_CACHE_LOCK = threading.Lock() _COST_ANALYSIS_PAYLOAD_CACHE: dict[tuple[Any, ...], dict[str, Any]] = {} COST_ANALYSIS_PAYLOAD_CACHE_TTL_SECONDS = 90.0 -COST_ANALYSIS_HANMAC_AGGREGATE_SCHEMA = "cost-analysis-period-v11-linked-balance" +COST_ANALYSIS_HANMAC_AGGREGATE_SCHEMA = "cost-analysis-period-v13-cumulative-profit-rate" app = FastAPI() app.add_middleware(GZipMiddleware, minimum_size=1200, compresslevel=5) @@ -670,7 +678,7 @@ def ensure_auth_schema(conn: Any) -> None: for permission_key, permission_name in permissions: conn.execute( text( - """ + f""" INSERT INTO app_permissions (permission_key, permission_name, description) VALUES (:permission_key, :permission_name, '') ON CONFLICT(permission_key) DO UPDATE SET @@ -952,6 +960,15 @@ def _auth_user_can(user: dict[str, Any] | None, permission: str) -> bool: return permission in set(user.get("permissions") or []) +def _auth_default_landing_for_user(user: dict[str, Any] | None) -> str: + if not user: + return "/" + for item in AUTH_NAV_ITEMS: + if _auth_user_can(user, str(item.get("permission") or "")): + return str(item.get("href") or "/") + return "/" + + def _auth_path_permission(path: str) -> str: if path == "/": return "dashboard" @@ -1054,6 +1071,22 @@ def authenticate_user(username: str, password: str) -> dict[str, Any] | None: return _auth_fetch_user(int(row["id"])) +def _format_kst_display_from_utc(value: Any) -> str: + text_value = normalize_text(value) + if not text_value: + return "" + try: + if text_value.endswith("Z"): + parsed = datetime.fromisoformat(text_value[:-1] + "+00:00") + else: + parsed = datetime.fromisoformat(text_value) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone(timedelta(hours=9))).strftime("%Y-%m-%d %H:%M:%S") + except Exception: + return text_value + + def list_admin_users() -> list[dict[str, Any]]: init_db() with engine.begin() as conn: @@ -1068,9 +1101,19 @@ def list_admin_users() -> list[dict[str, Any]]: u.is_admin, u.created_at, u.updated_at, + last_success.created_at AS last_login_at, COALESCE(GROUP_CONCAT(DISTINCT r.role_key), '') AS roles, COALESCE(GROUP_CONCAT(DISTINCT p.permission_key), '') AS direct_permissions FROM app_users AS u + LEFT JOIN ( + SELECT + user_id, + MAX(created_at) AS created_at + FROM app_login_events + WHERE success = 1 + AND user_id IS NOT NULL + GROUP BY user_id + ) AS last_success ON last_success.user_id = u.id LEFT JOIN app_user_roles AS ur ON ur.user_id = u.id LEFT JOIN app_roles AS r ON r.id = ur.role_id LEFT JOIN app_user_permissions AS up ON up.user_id = u.id @@ -1091,11 +1134,73 @@ def list_admin_users() -> list[dict[str, Any]]: "direct_permissions": [item for item in str(row["direct_permissions"] or "").split(",") if item], "created_at": str(row["created_at"] or ""), "updated_at": str(row["updated_at"] or ""), + "last_login_at": str(row["last_login_at"] or ""), + "last_login_at_kst": _format_kst_display_from_utc(row["last_login_at"]), } for row in rows ] +def list_admin_user_login_events(user_id: int, limit: int = 100) -> dict[str, Any]: + init_db() + limit = max(1, min(int(limit or 100), 300)) + with engine.begin() as conn: + user = conn.execute( + text( + """ + SELECT id, username, display_name + FROM app_users + WHERE id = :user_id + LIMIT 1 + """ + ), + {"user_id": int(user_id)}, + ).mappings().first() + if not user: + raise ValueError("사용자를 찾을 수 없습니다.") + rows = conn.execute( + text( + """ + SELECT + id, + user_id, + username, + success, + failure_reason, + ip_address, + user_agent, + created_at + FROM app_login_events + WHERE user_id = :user_id + OR username = :username + ORDER BY created_at DESC, id DESC + LIMIT :limit + """ + ), + {"user_id": int(user["id"]), "username": user["username"], "limit": limit}, + ).mappings().all() + return { + "user": { + "id": int(user["id"]), + "username": user["username"], + "display_name": user["display_name"] or user["username"], + }, + "events": [ + { + "id": int(row["id"]), + "username": row["username"] or "", + "success": bool(row["success"]), + "failure_reason": row["failure_reason"] or "", + "ip_address": row["ip_address"] or "", + "user_agent": row["user_agent"] or "", + "created_at": str(row["created_at"] or ""), + "created_at_kst": _format_kst_display_from_utc(row["created_at"]), + } + for row in rows + ], + } + + def list_admin_permission_options() -> list[dict[str, str]]: init_db() with engine.begin() as conn: @@ -1248,6 +1353,8 @@ async def auth_middleware(request: Request, call_next): return RedirectResponse(url=f"/login?next={_auth_redirect_target(request)}", status_code=303) permission = _auth_path_permission(path) if not _auth_user_can(user, permission): + if path == "/" and not _auth_is_api_request(request): + return RedirectResponse(url=_auth_default_landing_for_user(user), status_code=303) if _auth_is_api_request(request): return JSONResponse({"error": "forbidden"}, status_code=403) return HTMLResponse("

403

접근 권한이 없습니다.

", status_code=403) @@ -1452,12 +1559,35 @@ def _load_projection_group_counts( *, signature_like: str | None = None, ) -> tuple[dict[str, int], int]: + active_signature = "" + if signature_like and signature_like.startswith(f"{QUERY_PROJECTION_VERSION}|"): + try: + active_row = conn.execute( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = ? + LIMIT 1 + """, + (f"wehago_active_query_projection:{int(start_year)}:{int(end_year)}",), + ).fetchone() + if active_row and active_row[0]: + active_payload = json.loads(str(active_row[0] or "{}")) + if isinstance(active_payload, dict): + active_signature = normalize_text(active_payload.get("signature")) + except Exception: + active_signature = "" + if not active_signature.startswith(f"{QUERY_PROJECTION_VERSION}|"): + return {}, 0 where = [ "start_year <= ?", "end_year >= ?", ] params: list[Any] = [int(start_year), int(end_year)] - if signature_like: + if active_signature: + where = ["start_year = ?", "end_year = ?", "signature = ?"] + params = [int(start_year), int(end_year), active_signature] + elif signature_like: where.append("signature LIKE ?") params.append(signature_like) scope_row = conn.execute( @@ -1556,15 +1686,14 @@ def _fast_wehago_compare_summary_payload( for status_key, value in current_group_counts.items(): if status_key in counts: counts[status_key] = int(value or 0) - if "hanmac_unconnected" not in current_group_counts: - with engine.begin() as sqlalchemy_conn: - counts["hanmac_unconnected"] = len( - _load_hanmac_unconnected_source_groups( - sqlalchemy_conn, - int(start_year or 0), - int(end_year or 0), - ) + with engine.begin() as sqlalchemy_conn: + counts["hanmac_unconnected"] = len( + _load_hanmac_unconnected_source_groups( + sqlalchemy_conn, + int(start_year or 0), + int(end_year or 0), ) + ) if current_group_counts: scope_row = conn.execute( """ @@ -1672,6 +1801,18 @@ def _fast_wehago_compare_summary_payload( ).fetchone() counts[status_key] = int((row or [0])[0] or 0) + try: + resolved_counts, count_pending = get_dashboard_metric_counts_nonblocking( + engine, + int(start_year or 0), + int(end_year or 0), + ) + if any(int(resolved_counts.get(status_key, 0) or 0) for status_key in counts): + for status_key in counts: + counts[status_key] = int(resolved_counts.get(status_key, counts[status_key]) or 0) + except Exception: + count_pending = False + snapshot_state = {"ready": [], "stale": [], "missing": [], "queued": [], "running": [], "failed": []} status_rows = { int(row["fiscal_year"]): str(row["state"] or "") @@ -1690,6 +1831,7 @@ def _fast_wehago_compare_summary_payload( snapshot_state.setdefault(state, []) snapshot_state[state].append(year) pending = bool(snapshot_state["missing"] or snapshot_state["stale"] or snapshot_state["queued"] or snapshot_state["running"]) + pending = pending or bool(count_pending) last_action = None last_action_row = conn.execute( @@ -1736,10 +1878,19 @@ def _fast_wehago_compare_summary_payload( ("bridge_expense_review", "2단계 비교", ""), ) ] + with engine.begin() as sqlalchemy_conn: + final_status_summary = get_wehago_final_status_summary_from_conn( + sqlalchemy_conn, + start_year, + end_year, + counts, + ) + metric_sections = apply_wehago_final_status_counts_to_metric_sections(metric_sections, final_status_summary) return { "selected_start_year": start_year, "selected_end_year": end_year, "metric_sections": metric_sections, + "wehago_final_status_summary": final_status_summary, "last_action": last_action, "pending": pending, "snapshot_state": snapshot_state, @@ -2483,6 +2634,27 @@ def init_db() -> None: """ ) ) + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS wehago_benefit_category_overrides ( + ledger_row_id INTEGER PRIMARY KEY, + category TEXT NOT NULL DEFAULT '', + memo TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + ) + conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_wehago_benefit_category_overrides_updated + ON wehago_benefit_category_overrides (updated_at) + """ + ) + ) conn.execute( text( """ @@ -4866,8 +5038,12 @@ def _safe_next_url(value: Any) -> str: @app.get("/login") async def login_page(request: Request, next: str = ""): init_db() - if _auth_get_request_user(request): - return RedirectResponse(url=_safe_next_url(next), status_code=303) + current_user = _auth_get_request_user(request) + if current_user: + next_url = _safe_next_url(next) + if next_url == "/" and not _auth_user_can(current_user, "dashboard"): + next_url = _auth_default_landing_for_user(current_user) + return RedirectResponse(url=next_url, status_code=303) return templates.TemplateResponse( request, "login.html", @@ -4890,6 +5066,8 @@ async def login_submit(request: Request): {"request": request, "next_url": next_url, "error": "아이디 또는 비밀번호가 올바르지 않습니다."}, status_code=401, ) + if next_url == "/" and not _auth_user_can(user, "dashboard"): + next_url = _auth_default_landing_for_user(user) session_token = create_login_session(int(user["id"]), request) _auth_log_event(username, True, request, user_id=int(user["id"])) response = RedirectResponse(url=next_url, status_code=303) @@ -4920,6 +5098,14 @@ async def admin_users(request: Request, message: str = ""): return templates.TemplateResponse(request, "admin_users.html", context) +@app.get("/admin/users/{user_id}/login-events") +async def admin_user_login_events(user_id: int, limit: int = 100): + try: + return JSONResponse(list_admin_user_login_events(user_id, limit=limit)) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=404) + + @app.post("/admin/users") async def admin_users_save(request: Request): form = await request.form() @@ -6730,6 +6916,7 @@ BUSINESS_DATA_INCLUDE_TABLES = { "project_uncontracted_classification", "transactions", "wehago_compare_settings", + "wehago_benefit_category_overrides", "wehago_comparison_results", "wehago_ledger_rows", "wehago_manual_pair_matches", @@ -11315,6 +11502,762 @@ def base_context(request: Request, message: str = "") -> dict[str, Any]: } +WEHAGO_BENEFIT_DEFAULT_START_YEAR = 2023 +WEHAGO_BENEFIT_DEFAULT_END_YEAR = 2025 +WEHAGO_BENEFIT_CATEGORIES = ["급여성비용", "공통복지비", "부서/현장 운영비", "임원 개인성 비용", "경조사", "식대", "운동비", "기타"] +WEHAGO_BENEFIT_EXPORT_COLUMNS = [ + ("year", "연도"), + ("ledger_date", "일자"), + ("voucher_no", "전표번호"), + ("account_group", "구분"), + ("category", "분류"), + ("person_name", "개인/귀속"), + ("hanmac_member_grade", "직급"), + ("account_name", "계정"), + ("vendor_name", "거래처"), + ("debit", "차변금액"), + ("credit", "대변금액"), + ("net_amount", "순금액"), + ("amount", "금액"), + ("description", "적요"), + ("counterpart_details", "상대계정"), + ("basis", "분류근거"), + ("classification_review", "직급검토"), +] + +WEHAGO_EXECUTIVE_GRADES = {"대표", "회장", "부회장", "사장", "부사장", "전무", "상무", "이사"} +WEHAGO_CONGRAT_CONDOLENCE_KEYWORDS = ( + "경조", + "조의", + "부의", + "부고", + "근조", + "화환", + "축의", + "결혼", + "장례", + "상조", + "부친상", + "모친상", + "빙부", + "빙부상", + "빙모", + "빙모상", + "시부상", + "시모상", + "배우자상", + "자녀출산", + "아기출산", + "출산", + "칠순", + "팔순", + "회갑", + "환갑", + "돌잔치", +) + + +def _normalize_report_year(value: Any, fallback: int) -> int: + try: + year = int(value) + except Exception: + return fallback + if year < 2000 or year > 2100: + return fallback + return year + + +def _normalize_report_bool(value: Any) -> bool: + return normalize_text(value).lower() in {"1", "true", "y", "yes", "on", "포함"} + + +def _contains_any_keyword(text_value: str, keywords: tuple[str, ...]) -> str: + normalized = normalize_text(text_value).lower() + for keyword in keywords: + if keyword.lower() in normalized: + return keyword + return "" + + +def _normalize_wehago_vendor_name(value: Any) -> str: + vendor = normalize_text(value) + corrections = { + "임원/한형과": "임원/한형관", + "한형과": "한형관", + } + return corrections.get(vendor, vendor) + + +def _is_wehago_adjustment_or_reclass_row(row: Mapping[str, Any]) -> bool: + text_value = " ".join( + [ + normalize_text(row.get("account_code")), + normalize_text(row.get("account_name")), + normalize_text(row.get("vendor_name")), + normalize_text(row.get("description")), + ] + ) + return bool(_contains_any_keyword(text_value, ("대체", "원가", "손익"))) + + +def _infer_wehago_person_name(vendor_name: Any, description: Any) -> str: + vendor = _normalize_wehago_vendor_name(vendor_name) + if "/" in vendor: + tail = normalize_text(vendor.rsplit("/", 1)[-1]) + if tail: + return tail + if re.fullmatch(r"[가-힣]{2,5}[A-Za-z]?", vendor): + return vendor + desc = normalize_text(description) + match = re.search(r"(?:대표|사장|부사장|전무|상무|이사|임원)?\s*([가-힣]{2,4}[A-Za-z]?)", desc) + if match and any(title in desc for title in ("대표", "사장", "부사장", "전무", "상무", "이사", "임원")): + return match.group(1) + return vendor or "미지정" + + +def _hanmac_member_grade_lookup_signature() -> str: + init_db() + with engine.begin() as conn: + db_signature = conn.execute( + text( + """ + SELECT COUNT(*) || ':' || COALESCE(MAX(updated_at), '') + FROM hanmac_aggregate_query_metrics + WHERE view_mode = 'member' + """ + ) + ).scalar() + joint_cache_path = BASE_DIR / "static" / "hanmac-joint-members-cache.json" + try: + stat = joint_cache_path.stat() + file_signature = f"{stat.st_mtime_ns}:{stat.st_size}" + except Exception: + file_signature = "" + return hashlib.sha1(f"{db_signature or ''}|{file_signature}".encode("utf-8")).hexdigest() + + +def _load_hanmac_member_grade_lookup() -> dict[str, dict[str, Any]]: + return dict(_load_hanmac_member_grade_lookup_cached(_hanmac_member_grade_lookup_signature())) + + +@lru_cache(maxsize=4) +def _load_hanmac_member_grade_lookup_cached(_signature: str) -> dict[str, dict[str, Any]]: + init_db() + with engine.begin() as conn: + cache_keys = [ + normalize_text(row[0]) + for row in conn.execute( + text( + """ + SELECT cache_key + FROM hanmac_aggregate_query_metrics + WHERE view_mode = 'member' + ORDER BY updated_at DESC + LIMIT 8 + """ + ) + ).fetchall() + if normalize_text(row[0]) + ] + row_items: list[str] = [] + for cache_key in cache_keys: + row_items.extend( + str(item or "") + for item in conn.execute( + text( + """ + SELECT row_json + FROM hanmac_aggregate_query_rows + WHERE cache_key = :cache_key + AND ( + row_json LIKE '%member_grade%' + OR row_json LIKE '%"grade"%' + OR row_json LIKE '%"position"%' + OR row_json LIKE '%"rank"%' + ) + ORDER BY row_index + """ + ), + {"cache_key": cache_key}, + ).scalars().all() + ) + lookup: dict[str, dict[str, Any]] = {} + + def add_member_row(row: dict[str, Any]) -> None: + member_name = normalize_text(row.get("member_name") or row.get("name")) + if not member_name: + return + member_grade = _normalize_labor_grade_name( + row.get("member_grade") + or row.get("grade") + or row.get("position") + or row.get("rank") + ) + member_key = _hanmac_normalize_person_name(member_name) + if not member_key: + return + existing = lookup.get(member_key) + if existing and existing.get("member_grade") and not member_grade: + return + lookup[member_key] = { + "member_name": member_name, + "member_no": normalize_text(row.get("member_no")), + "member_grade": member_grade, + "dept_name": normalize_text(row.get("dept_name")), + "status": normalize_text(row.get("status")), + } + + for item in row_items: + try: + row = json.loads(item or "{}") + except Exception: + continue + if isinstance(row, dict): + add_member_row(row) + + joint_cache_path = BASE_DIR / "static" / "hanmac-joint-members-cache.json" + try: + joint_payload = json.loads(joint_cache_path.read_text(encoding="utf-8")) + except Exception: + joint_payload = {} + if isinstance(joint_payload, dict): + payload_items = [] + by_key = joint_payload.get("by_key") + if isinstance(by_key, dict): + payload_items.extend(value for value in by_key.values() if isinstance(value, dict)) + latest = joint_payload.get("latest") + if isinstance(latest, dict): + payload_items.append(latest) + for payload_item in payload_items: + for row in payload_item.get("joint_members") or []: + if isinstance(row, dict): + add_member_row(row) + return lookup + + +def _lookup_hanmac_member_grade(person_name: Any, member_lookup: dict[str, dict[str, Any]]) -> dict[str, Any]: + person_key = _hanmac_normalize_person_name(person_name) + if not person_key: + return {} + return member_lookup.get(person_key) or {} + + +def _is_wehago_executive_grade(member_grade: Any) -> bool: + grade = _normalize_labor_grade_name(member_grade) + return bool(grade and (grade in WEHAGO_EXECUTIVE_GRADES or any(item in grade for item in WEHAGO_EXECUTIVE_GRADES))) + + +def save_wehago_benefit_category_override(ledger_row_id: Any, category: Any) -> dict[str, Any]: + row_id = int(ledger_row_id or 0) + normalized_category = normalize_text(category) + if row_id <= 0: + raise ValueError("전표 행 ID가 올바르지 않습니다.") + if normalized_category not in set(WEHAGO_BENEFIT_CATEGORIES): + raise ValueError("분류 값이 올바르지 않습니다.") + init_db() + init_wehago_compare_db(engine) + with engine.begin() as conn: + exists = conn.execute( + text("SELECT 1 FROM wehago_ledger_rows WHERE id = :row_id LIMIT 1"), + {"row_id": row_id}, + ).scalar() + if not exists: + raise ValueError("저장할 전표 행을 찾을 수 없습니다.") + conn.execute( + text( + """ + INSERT INTO wehago_benefit_category_overrides ( + ledger_row_id, category, updated_at + ) VALUES ( + :ledger_row_id, :category, CURRENT_TIMESTAMP + ) + ON CONFLICT(ledger_row_id) DO UPDATE SET + category = excluded.category, + updated_at = CURRENT_TIMESTAMP + """ + ), + {"ledger_row_id": row_id, "category": normalized_category}, + ) + return {"ok": True, "ledger_row_id": row_id, "category": normalized_category} + + +def _classify_wehago_benefit_row(row: dict[str, Any], member_grade: str = "") -> tuple[str, str, str]: + account_code = normalize_text(row.get("account_code")) + account_name = normalize_text(row.get("account_name")) + vendor_name = _normalize_wehago_vendor_name(row.get("vendor_name")) + description = normalize_text(row.get("description")) + text_value = " ".join([account_code, account_name, vendor_name, description]) + congrat_keyword = _contains_any_keyword(text_value, WEHAGO_CONGRAT_CONDOLENCE_KEYWORDS) + + if "접대" in account_name or account_code.startswith("813"): + account_group = "접대비" + rules = ( + ("경조사", WEHAGO_CONGRAT_CONDOLENCE_KEYWORDS), + ("운동비", ("골프", "운동", "체력", "연습장", "스포츠", "피트니스")), + ("식대", ("식대", "식사", "오찬", "만찬", "점심", "저녁", "회식", "음식", "식당", "카페", "커피", "주점")), + ) + for category, keywords in rules: + matched = _contains_any_keyword(text_value, keywords) + if matched: + return account_group, category, matched + return account_group, "기타", "" + + account_group = "복리후생비" + payroll_keyword = _contains_any_keyword(text_value, ("건강보험료", "장기요양보험료", "고용보험료", "산재보험료")) + if payroll_keyword: + return account_group, "급여성비용", payroll_keyword + + executive_keyword = _contains_any_keyword(text_value, ("체력", "운동", "헬스", "골프", "골프연습", "연습장", "피트니스")) + grade_is_executive = _is_wehago_executive_grade(member_grade) + vendor_is_executive = grade_is_executive or vendor_name.startswith("임원/") or _contains_any_keyword(text_value, ("대표", "사장", "부사장", "전무", "상무", "이사")) + if congrat_keyword: + return account_group, "경조사", congrat_keyword + + meal_keyword = _contains_any_keyword(text_value, ("식대", "식사", "오찬", "만찬", "점심", "저녁")) + if executive_keyword or (vendor_is_executive and meal_keyword): + return account_group, "임원 개인성 비용", executive_keyword or meal_keyword or member_grade or "임원" + + common_keyword = _contains_any_keyword( + text_value, + ( + "전직원", + "전 직원", + "임직원", + "전체", + "명절", + "선물", + "건강보험", + "고용보험", + "산재", + "장기요양", + "국민연금", + "보험료", + "복지", + "건강검진", + "단체", + "창립", + ), + ) + if common_keyword: + return account_group, "공통복지비", common_keyword + + operation_keyword = _contains_any_keyword( + text_value, + ( + "부서", + "현장", + "회식", + "간식", + "식대", + "식사", + "점심", + "저녁", + "야근", + "야식", + "합사", + "사무실", + "회의", + "워크샵", + "워크숍", + "송년회", + ), + ) + if operation_keyword: + return account_group, "부서/현장 운영비", operation_keyword + return account_group, "기타", "" + + +def _wehago_voucher_key(row: Mapping[str, Any]) -> tuple[int, str, str]: + return ( + int(row.get("year") or row.get("fiscal_year") or 0), + normalize_text(row.get("ledger_date")), + normalize_text(row.get("voucher_no")), + ) + + +def _format_wehago_counterpart_details(row: Mapping[str, Any], voucher_lines: Sequence[Mapping[str, Any]]) -> str: + row_id = int(row.get("ledger_row_id") or 0) + lines: list[str] = [] + for line in voucher_lines: + line_id = int(line.get("ledger_row_id") or 0) + if row_id and line_id == row_id: + continue + debit = normalize_amount(line.get("debit")) + credit = normalize_amount(line.get("credit")) + side = "차" if debit else "대" + amount = debit if debit else credit + pieces = [ + f"[{side}]", + normalize_text(line.get("account_name")) or normalize_text(line.get("account_code")), + ] + vendor_name = normalize_text(line.get("vendor_name")) + if vendor_name: + pieces.append(vendor_name) + if amount: + pieces.append(f"{amount:,.0f}") + description = normalize_text(line.get("description")) + if description: + pieces.append(description) + lines.append(" ".join(piece for piece in pieces if piece)) + return " / ".join(lines) + + +def _format_wehago_counterpart_tooltip(counterpart_details: Any) -> str: + return normalize_text(counterpart_details).replace(" / [", "\n[") + + +def get_wehago_benefit_entertainment_report( + start_year: int = WEHAGO_BENEFIT_DEFAULT_START_YEAR, + end_year: int = WEHAGO_BENEFIT_DEFAULT_END_YEAR, + account_group: str = "all", + category: str = "all", + person_keyword: str = "", + desc_keyword: str = "", + include_adjustments: bool = False, + limit: int | None = 300, +) -> dict[str, Any]: + init_db() + init_wehago_compare_db(engine) + start_year = _normalize_report_year(start_year, WEHAGO_BENEFIT_DEFAULT_START_YEAR) + end_year = _normalize_report_year(end_year, WEHAGO_BENEFIT_DEFAULT_END_YEAR) + if start_year > end_year: + start_year, end_year = end_year, start_year + + include_adjustments = bool(include_adjustments) + adjustment_sql = "" + if not include_adjustments: + adjustment_sql = """ + AND COALESCE(account_name, '') NOT LIKE '%대체%' + AND COALESCE(account_name, '') NOT LIKE '%원가%' + AND COALESCE(account_name, '') NOT LIKE '%손익%' + AND COALESCE(vendor_name, '') NOT LIKE '%대체%' + AND COALESCE(vendor_name, '') NOT LIKE '%원가%' + AND COALESCE(vendor_name, '') NOT LIKE '%손익%' + AND COALESCE(description, '') NOT LIKE '%대체%' + AND COALESCE(description, '') NOT LIKE '%원가%' + AND COALESCE(description, '') NOT LIKE '%손익%' + """ + params = {"start_year": start_year, "end_year": end_year} + with engine.begin() as conn: + source_rows = conn.execute( + text( + f""" + SELECT + id AS ledger_row_id, + fiscal_year AS year, + COALESCE(ledger_date, '') AS ledger_date, + COALESCE(voucher_no, '') AS voucher_no, + COALESCE(account_code, '') AS account_code, + COALESCE(account_name, '') AS account_name, + COALESCE(vendor_name, '') AS vendor_name, + COALESCE(debit, 0) AS debit, + COALESCE(credit, 0) AS credit, + COALESCE(description, '') AS description + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(debit, 0) > 0 + AND ( + COALESCE(account_name, '') LIKE '%복리후생%' + OR COALESCE(account_name, '') LIKE '%접대%' + OR COALESCE(account_code, '') IN ('611', '811', '813') + ) + {adjustment_sql} + ORDER BY fiscal_year, ledger_date, voucher_no, id + """ + ), + params, + ).mappings().all() + override_rows = conn.execute( + text( + """ + SELECT ledger_row_id, category + FROM wehago_benefit_category_overrides + """ + ) + ).mappings().all() + voucher_line_rows: list[Mapping[str, Any]] = [] + voucher_keys = sorted({_wehago_voucher_key(row) for row in source_rows if normalize_text(row.get("voucher_no"))}) + with engine.begin() as conn: + for chunk_start in range(0, len(voucher_keys), 250): + chunk = voucher_keys[chunk_start : chunk_start + 250] + if not chunk: + continue + chunk_params: dict[str, Any] = {} + values_sql: list[str] = [] + for index, (year, ledger_date, voucher_no) in enumerate(chunk): + year_key = f"year_{index}" + date_key = f"ledger_date_{index}" + voucher_key = f"voucher_no_{index}" + values_sql.append(f"(:{year_key}, :{date_key}, :{voucher_key})") + chunk_params[year_key] = year + chunk_params[date_key] = ledger_date + chunk_params[voucher_key] = voucher_no + voucher_line_rows.extend( + conn.execute( + text( + f""" + WITH target_keys(year, ledger_date, voucher_no) AS ( + VALUES {", ".join(values_sql)} + ) + SELECT + line.id AS ledger_row_id, + line.fiscal_year AS year, + COALESCE(line.ledger_date, '') AS ledger_date, + COALESCE(line.voucher_no, '') AS voucher_no, + COALESCE(line.account_code, '') AS account_code, + COALESCE(line.account_name, '') AS account_name, + COALESCE(line.vendor_name, '') AS vendor_name, + COALESCE(line.debit, 0) AS debit, + COALESCE(line.credit, 0) AS credit, + COALESCE(line.description, '') AS description + FROM wehago_ledger_rows AS line + JOIN target_keys AS target + ON target.year = line.fiscal_year + AND target.ledger_date = COALESCE(line.ledger_date, '') + AND target.voucher_no = COALESCE(line.voucher_no, '') + ORDER BY line.fiscal_year, line.ledger_date, line.voucher_no, line.id + """ + ), + chunk_params, + ).mappings().all() + ) + voucher_lines_by_key: dict[tuple[int, str, str], list[dict[str, Any]]] = {} + for voucher_line in voucher_line_rows: + line_item = dict(voucher_line) + voucher_lines_by_key.setdefault(_wehago_voucher_key(line_item), []).append(line_item) + category_overrides = { + int(row["ledger_row_id"]): normalize_text(row["category"]) + for row in override_rows + if row.get("ledger_row_id") is not None + } + + account_group_filter = normalize_text(account_group) + category_filter = normalize_text(category) + person_filter = normalize_text(person_keyword).lower() + desc_filter = normalize_text(desc_keyword).lower() + member_grade_lookup = _load_hanmac_member_grade_lookup() + detail_rows: list[dict[str, Any]] = [] + for row in source_rows: + item = dict(row) + item["vendor_name"] = _normalize_wehago_vendor_name(item.get("vendor_name")) + person_name = _infer_wehago_person_name(item.get("vendor_name"), item.get("description")) + member_record = _lookup_hanmac_member_grade(person_name, member_grade_lookup) + member_grade = normalize_text(member_record.get("member_grade")) + row_group, row_category, basis = _classify_wehago_benefit_row(item, member_grade=member_grade) + ledger_row_id = int(item.get("ledger_row_id") or 0) + manual_category = category_overrides.get(ledger_row_id, "") + if manual_category: + row_category = manual_category + basis = "수동저장" + if account_group_filter not in {"", "all", "전체"} and row_group != account_group_filter: + continue + if category_filter not in {"", "all", "전체"} and row_category != category_filter: + continue + if person_filter and person_filter not in normalize_text(person_name).lower() and person_filter not in normalize_text(item.get("vendor_name")).lower(): + continue + searchable_desc = " ".join([normalize_text(item.get("description")), normalize_text(item.get("vendor_name")), normalize_text(item.get("account_name"))]).lower() + if desc_filter and desc_filter not in searchable_desc: + continue + debit = normalize_amount(item.get("debit")) + credit = normalize_amount(item.get("credit")) + if debit <= 0: + continue + if not include_adjustments and _is_wehago_adjustment_or_reclass_row(item): + continue + net_amount = debit - credit + amount = debit + counterpart_details = _format_wehago_counterpart_details(item, voucher_lines_by_key.get(_wehago_voucher_key(item), [])) + counterpart_tooltip = _format_wehago_counterpart_tooltip(counterpart_details) + classification_review = "" + if member_grade: + classification_review = f"hanmac DB_external 직급 확인: {member_grade}" + if _is_wehago_executive_grade(member_grade) and row_group == "복리후생비": + classification_review += " / 임원급 기준 검토" + detail_rows.append( + { + **item, + "ledger_row_id": ledger_row_id, + "account_group": row_group, + "category": row_category, + "manual_category": manual_category, + "is_manual_category": bool(manual_category), + "person_name": person_name, + "hanmac_member_no": normalize_text(member_record.get("member_no")), + "hanmac_member_grade": member_grade, + "hanmac_dept_name": normalize_text(member_record.get("dept_name")), + "debit": debit, + "credit": credit, + "net_amount": net_amount, + "amount": amount, + "counterpart_details": counterpart_details, + "counterpart_tooltip": counterpart_tooltip, + "basis": basis or "기타", + "classification_review": classification_review, + } + ) + + summary_map: dict[tuple[str, str, str], dict[str, Any]] = {} + category_map: dict[tuple[str, str], dict[str, Any]] = {} + yearly_map: dict[tuple[int, str], dict[str, Any]] = {} + category_year_map: dict[tuple[str, str], dict[str, Any]] = {} + report_years = list(range(start_year, end_year + 1)) + for item in detail_rows: + vendor_name = normalize_text(item.get("vendor_name")) or "미지정" + person_name = normalize_text(item.get("person_name")) + summary_key = (item["account_group"], item["category"], vendor_name) + summary = summary_map.setdefault( + summary_key, + { + "account_group": item["account_group"], + "category": item["category"], + "vendor_name": vendor_name, + "person_name": person_name, + "person_names": set(), + "hanmac_member_grade": item.get("hanmac_member_grade", ""), + "amount": 0.0, + "debit": 0.0, + "credit": 0.0, + "row_count": 0, + "last_date": "", + }, + ) + if person_name: + summary["person_names"].add(person_name) + if not summary.get("hanmac_member_grade") and item.get("hanmac_member_grade"): + summary["hanmac_member_grade"] = item.get("hanmac_member_grade", "") + summary["amount"] += item["amount"] + summary["debit"] += item["debit"] + summary["credit"] += item["credit"] + summary["row_count"] += 1 + summary["last_date"] = max(summary["last_date"], normalize_text(item.get("ledger_date"))) + + category_key = (item["account_group"], item["category"]) + category_summary = category_map.setdefault(category_key, {"account_group": item["account_group"], "category": item["category"], "amount": 0.0, "row_count": 0}) + category_summary["amount"] += item["amount"] + category_summary["row_count"] += 1 + + category_year_summary = category_year_map.setdefault( + category_key, + { + "account_group": item["account_group"], + "category": item["category"], + "year_amounts": {year: 0.0 for year in report_years}, + "total_amount": 0.0, + "row_count": 0, + }, + ) + item_year = int(item.get("year") or 0) + if item_year not in category_year_summary["year_amounts"]: + category_year_summary["year_amounts"][item_year] = 0.0 + category_year_summary["year_amounts"][item_year] += item["amount"] + category_year_summary["total_amount"] += item["amount"] + category_year_summary["row_count"] += 1 + + yearly_key = (int(item.get("year") or 0), item["account_group"]) + yearly_summary = yearly_map.setdefault(yearly_key, {"year": int(item.get("year") or 0), "account_group": item["account_group"], "amount": 0.0, "row_count": 0}) + yearly_summary["amount"] += item["amount"] + yearly_summary["row_count"] += 1 + + summary_rows: list[dict[str, Any]] = [] + for row in summary_map.values(): + person_names = sorted(name for name in row.pop("person_names", set()) if name) + row["person_name"] = ", ".join(person_names[:4]) + (" 외" if len(person_names) > 4 else "") + summary_rows.append(row) + vendor_total_amounts: dict[str, float] = {} + for row in summary_rows: + vendor_name = normalize_text(row.get("vendor_name")) or "미지정" + vendor_total_amounts[vendor_name] = vendor_total_amounts.get(vendor_name, 0.0) + float(row.get("amount") or 0.0) + + def summary_sort_key(row: Mapping[str, Any]) -> tuple[float, str, float, str, str]: + vendor_name = normalize_text(row.get("vendor_name")) or "미지정" + return ( + -vendor_total_amounts.get(vendor_name, 0.0), + vendor_name, + -float(row.get("amount") or 0.0), + normalize_text(row.get("account_group")), + normalize_text(row.get("category")), + ) + + sorted_summary_rows = sorted(summary_rows, key=summary_sort_key) + category_year_summary_rows = sorted( + category_year_map.values(), + key=lambda row: (normalize_text(row.get("account_group")), -float(row.get("total_amount") or 0), normalize_text(row.get("category"))), + ) + all_detail_rows = sorted(detail_rows, key=lambda row: (row.get("year") or 0, row.get("ledger_date") or "", row.get("voucher_no") or "")) + shown_detail_rows = all_detail_rows if limit is None else all_detail_rows[: max(0, int(limit))] + total_amount = sum(row["amount"] for row in all_detail_rows) + return { + "filters": { + "start_year": start_year, + "end_year": end_year, + "account_group": account_group_filter or "all", + "category": category_filter or "all", + "person_keyword": normalize_text(person_keyword), + "desc_keyword": normalize_text(desc_keyword), + "include_adjustments": include_adjustments, + }, + "total_amount": total_amount, + "row_count": len(all_detail_rows), + "shown_count": len(shown_detail_rows), + "detail_rows": shown_detail_rows, + "all_detail_rows": all_detail_rows, + "vendor_summary_rows": sorted_summary_rows, + "person_summary_rows": sorted_summary_rows, + "category_summary_rows": sorted(category_map.values(), key=lambda row: (row["account_group"], -row["amount"], row["category"])), + "yearly_summary_rows": sorted(yearly_map.values(), key=lambda row: (row["year"], row["account_group"])), + "category_year_summary_rows": category_year_summary_rows, + "summary_years": report_years, + "category_options": WEHAGO_BENEFIT_CATEGORIES, + "account_group_options": ["복리후생비", "접대비"], + "hanmac_grade_match_count": sum(1 for row in all_detail_rows if row.get("hanmac_member_grade")), + "hanmac_grade_lookup_count": len(member_grade_lookup), + } + + +def export_wehago_benefit_entertainment_xlsx(report: dict[str, Any]) -> tuple[str, bytes]: + workbook = Workbook() + worksheet = workbook.active + worksheet.title = "상세" + header_fill = PatternFill("solid", fgColor="1F2937") + header_font = Font(color="FFFFFF", bold=True) + worksheet.append([label for _, label in WEHAGO_BENEFIT_EXPORT_COLUMNS]) + for cell in worksheet[1]: + cell.fill = header_fill + cell.font = header_font + cell.alignment = Alignment(horizontal="center") + for row in report["all_detail_rows"]: + worksheet.append([row.get(field, "") for field, _ in WEHAGO_BENEFIT_EXPORT_COLUMNS]) + worksheet.freeze_panes = "A2" + worksheet.auto_filter.ref = f"A1:Q{max(len(report['all_detail_rows']) + 1, 1)}" + widths = [8, 12, 12, 14, 18, 16, 10, 18, 24, 14, 14, 14, 14, 46, 54, 14, 28] + for index, width in enumerate(widths, start=1): + worksheet.column_dimensions[get_column_letter(index)].width = width + for row in worksheet.iter_rows(min_row=2, min_col=10, max_col=13): + for cell in row: + cell.number_format = '#,##0' + + summary_sheet = workbook.create_sheet(title="거래처별 요약") + summary_sheet.append(["구분", "분류", "거래처", "개인/귀속", "직급", "금액", "차변금액", "대변금액", "건수", "최근일자"]) + for cell in summary_sheet[1]: + cell.fill = header_fill + cell.font = header_font + cell.alignment = Alignment(horizontal="center") + for row in report["vendor_summary_rows"]: + summary_sheet.append([row.get("account_group"), row.get("category"), row.get("vendor_name"), row.get("person_name"), row.get("hanmac_member_grade"), row.get("amount"), row.get("debit"), row.get("credit"), row.get("row_count"), row.get("last_date")]) + summary_sheet.freeze_panes = "A2" + for column, width in zip("ABCDEFGHIJ", [14, 18, 24, 18, 10, 14, 14, 14, 10, 12], strict=False): + summary_sheet.column_dimensions[column].width = width + for row in summary_sheet.iter_rows(min_row=2, min_col=6, max_col=8): + for cell in row: + cell.number_format = '#,##0' + + buffer = BytesIO() + workbook.save(buffer) + filters = report["filters"] + file_name = f"wehago_benefit_entertainment_{filters['start_year']}_{filters['end_year']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" + return file_name, buffer.getvalue() + + def ensure_hmbiz_process_db() -> None: if HMBIZ_PROCESS_DB_PATH.exists(): return @@ -12718,6 +13661,154 @@ def _cost_analysis_load_hanmac_project_hours_by_year( return result +def _cost_analysis_load_hanmac_hours_and_labor_by_year( + start_date: date, + end_date: date, + project_meta: dict[str, dict[str, Any]], + allowed_codes: set[str] | None = None, +) -> tuple[dict[int, dict[str, dict[str, float]]], dict[int, dict[str, dict[str, float]]]]: + alias_to_code, title_to_codes = _cost_analysis_build_hanmac_matchers(project_meta) + metric, row_items = _cost_analysis_load_hanmac_member_rows(start_date, end_date, prefer_member_grade=True) + if not metric: + return {}, {} + + rates_by_year = _parse_labor_rates_json(get_shared_exec_labor_rates_json()) + if not rates_by_year: + rates_by_year = _parse_labor_rates_json(json.dumps(DEFAULT_EXEC_LABOR_RATES, ensure_ascii=False)) + completion_dates = _cost_analysis_get_completion_billing_dates() + hours_result: dict[int, dict[str, dict[str, float]]] = {} + labor_result: dict[int, dict[str, dict[str, float]]] = {} + resolve_cache: dict[tuple[str, str, str], list[str]] = {} + + def add_project(project: dict[str, Any], work_date_text: Any, hours: float, member_grade: str = "") -> None: + if hours <= 0: + return + work_date = _parse_iso_date(work_date_text) + if work_date and (work_date < start_date or work_date > end_date): + return + effective_date = work_date or start_date + resolve_key = ( + normalize_text(project.get("project_code")).upper(), + "|".join(normalize_text(value).upper() for value in (project.get("equivalent_project_codes") or [])), + f"{normalize_project_title_for_linking(project.get('project_name'))}|{effective_date.isoformat()}", + ) + if resolve_key in resolve_cache: + codes = resolve_cache[resolve_key] + else: + codes = _cost_analysis_resolve_hanmac_project_codes(project, effective_date, alias_to_code, title_to_codes, project_meta) + resolve_cache[resolve_key] = codes + if not codes: + return + + split_hours = hours / len(codes) + year = effective_date.year + year_text = str(year) + for code in codes: + normalized_code = normalize_text(code).upper() + if allowed_codes is not None and normalized_code not in allowed_codes: + continue + phase = _cost_analysis_phase_for_transaction(normalized_code, effective_date.isoformat(), completion_dates) + hours_result.setdefault(year, {}).setdefault( + normalized_code, + {"pre": 0.0, "during": 0.0, "post": 0.0}, + )[phase] += split_hours + + if not member_grade: + continue + rate = _resolve_labor_rate( + rates_by_year, + member_grade, + year_text, + year_text, + (project_meta.get(normalized_code) or {}).get("project_type"), + ) + if rate <= 0: + continue + labor_result.setdefault(year, {}).setdefault( + normalized_code, + {"pre": 0.0, "during": 0.0, "post": 0.0}, + )[phase] += rate * split_hours + + for row in row_items: + member_grade = _normalize_labor_grade_name( + row.get("member_grade") + or row.get("grade") + or row.get("position") + or row.get("rank") + ) + details = row.get("aggregate_details") if isinstance(row.get("aggregate_details"), dict) else {} + for detail in details.get("regular_hours") or []: + projects = detail.get("projects") if isinstance(detail.get("projects"), list) else [] + raw_total = sum(normalize_amount(project.get("hours")) for project in projects) + recognized_total = normalize_amount(detail.get("regular_hours")) + for project in projects: + raw_hours = normalize_amount(project.get("hours")) + hours = recognized_total * raw_hours / raw_total if raw_total > 0 and recognized_total > 0 else raw_hours + add_project(project, detail.get("work_date"), hours, member_grade) + for detail in details.get("overtime_hours") or []: + add_project(detail, detail.get("work_date"), normalize_amount(detail.get("overtime_hours")), member_grade) + for detail in details.get("holiday_hours") or []: + projects = detail.get("projects") if isinstance(detail.get("projects"), list) else [] + if projects: + raw_total = sum(normalize_amount(project.get("hours")) for project in projects) + recognized_total = normalize_amount(detail.get("holiday_hours")) + for project in projects: + raw_hours = normalize_amount(project.get("hours")) + hours = recognized_total * raw_hours / raw_total if raw_total > 0 and recognized_total > 0 else raw_hours + add_project(project, detail.get("work_date"), hours, member_grade) + else: + add_project(detail, detail.get("work_date"), normalize_amount(detail.get("holiday_hours")), member_grade) + return hours_result, labor_result + + +def _cost_analysis_load_hanmac_hours_and_labor_yearly( + start_date: date, + end_date: date, + project_meta: dict[str, dict[str, Any]], + allowed_codes: set[str] | None = None, +) -> tuple[dict[int, dict[str, dict[str, float]]], dict[int, dict[str, dict[str, float]]]]: + hours_result: dict[int, dict[str, dict[str, float]]] = {} + labor_result: dict[int, dict[str, dict[str, float]]] = {} + + def merge( + hours_by_year: dict[int, dict[str, dict[str, float]]], + labor_by_year: dict[int, dict[str, dict[str, float]]], + ) -> None: + for year, code_map in hours_by_year.items(): + for code, phase_hours in code_map.items(): + target = hours_result.setdefault(year, {}).setdefault(code, {"pre": 0.0, "during": 0.0, "post": 0.0}) + for phase, hours in phase_hours.items(): + if phase in target: + target[phase] += normalize_amount(hours) + for year, code_map in labor_by_year.items(): + for code, phase_amounts in code_map.items(): + target = labor_result.setdefault(year, {}).setdefault(code, {"pre": 0.0, "during": 0.0, "post": 0.0}) + for phase, amount in phase_amounts.items(): + if phase in target: + target[phase] += normalize_amount(amount) + + prefix_metric = _cost_analysis_select_hanmac_prefix_metric(start_date, end_date, prefer_member_grade=True) + prefix_end = _parse_iso_date((prefix_metric or {}).get("end_date")) + if prefix_metric and prefix_end and prefix_end >= start_date: + merge(*_cost_analysis_load_hanmac_hours_and_labor_by_year( + start_date, + min(prefix_end, end_date), + project_meta, + allowed_codes, + )) + start_date = min(prefix_end, end_date) + timedelta(days=1) + if start_date > end_date: + return hours_result, labor_result + for year_slice in _iter_year_slices(start_date, end_date): + merge(*_cost_analysis_load_hanmac_hours_and_labor_by_year( + year_slice["start"], + year_slice["end"], + project_meta, + allowed_codes, + )) + return hours_result, labor_result + + def _cost_analysis_get_annual_hanmac_total_hours(year: int) -> float: year_start = date(year, 1, 1).isoformat() year_end = date(year, 12, 31).isoformat() @@ -12786,6 +13877,17 @@ def _cost_analysis_row_template(code: str, meta: dict[str, Any], selected_year: "contract_amount": contract_amount, "billing_amount": 0.0, "collection_amount": 0.0, + "period_billing_amount": 0.0, + "period_collection_amount": 0.0, + "period_revenue_amount": 0.0, + "period_cost_total": 0.0, + "period_sga_total": 0.0, + "period_sales_total": 0.0, + "period_total_cost": 0.0, + "period_profit_amount": 0.0, + "period_revenue_profit_rate": 0.0, + "period_collection_profit_rate": 0.0, + "cumulative_profit_rate": 0.0, "contract_balance_amount": contract_amount, "collection_rate": 0.0, "revenue_amount": 0.0, @@ -12822,6 +13924,12 @@ def _cost_analysis_finalize_row(row: dict[str, Any]) -> None: row["sales_total"] = sales_total row["total_cost"] = cost_total + sga_total + sales_total row["profit_amount"] = normalize_amount(row.get("revenue_amount")) - row["total_cost"] + row["period_revenue_amount"] = normalize_amount(row.get("period_revenue_amount")) + row["period_cost_total"] = cost_total + row["period_sga_total"] = sga_total + row["period_sales_total"] = sales_total + row["period_total_cost"] = row["period_cost_total"] + row["period_sga_total"] + row["period_sales_total"] + row["period_profit_amount"] = row["period_revenue_amount"] - row["period_total_cost"] row["contract_balance_amount"] = max( normalize_amount(row.get("contract_amount")) - normalize_amount(row.get("collection_amount")), 0.0, @@ -12830,6 +13938,12 @@ def _cost_analysis_finalize_row(row: dict[str, Any]) -> None: row["contract_profit_rate"] = _safe_ratio(row.get("profit_amount"), row.get("contract_amount")) row["revenue_profit_rate"] = _safe_ratio(row.get("profit_amount"), row.get("revenue_amount")) row["collection_profit_rate"] = _safe_ratio(row.get("profit_amount"), row.get("collection_amount")) + row["period_revenue_profit_rate"] = _safe_ratio(row.get("period_profit_amount"), row.get("period_revenue_amount")) + row["period_collection_profit_rate"] = _safe_ratio(row.get("period_profit_amount"), row.get("period_collection_amount")) + row["cumulative_profit_rate"] = _safe_ratio( + normalize_amount(row.get("collection_amount")) - row["total_cost"], + row.get("collection_amount"), + ) def _cost_analysis_active_in_year(meta: dict[str, Any], year: int) -> bool: @@ -12902,9 +14016,62 @@ def _cost_analysis_get_link_representative_map() -> dict[str, str]: representative = total_codes[0] for code in component: representative_map[code] = representative + representative_map.update(_cost_analysis_infer_yz_link_representatives(representative_map)) return representative_map +def _cost_analysis_infer_yz_link_representatives(existing_representative_map: dict[str, str]) -> dict[str, str]: + project_meta = _cost_analysis_get_project_meta() + representative_titles: dict[str, set[str]] = {} + for code, representative in existing_representative_map.items(): + normalized_code = normalize_text(code).upper() + normalized_representative = normalize_text(representative).upper() + if not normalized_code or not normalized_representative or normalized_code == normalized_representative: + continue + title = normalize_project_title_for_linking((project_meta.get(normalized_code) or {}).get("support_dept_name")) + if title: + representative_titles.setdefault(normalized_representative, set()).add(title) + for representative, meta in project_meta.items(): + normalized_representative = normalize_text(representative).upper() + if normalized_representative[:1] not in {"0", "9"}: + continue + title = normalize_project_title_for_linking(meta.get("support_dept_name")) + if title: + representative_titles.setdefault(normalized_representative, set()).add(title) + + inferred: dict[str, str] = {} + for code, meta in project_meta.items(): + normalized_code = normalize_text(code).upper() + if ( + not normalized_code + or normalized_code in existing_representative_map + or normalized_code[:1] not in {"Y", "Z"} + ): + continue + title = normalize_project_title_for_linking(meta.get("support_dept_name")) + if not title: + continue + matches: list[tuple[int, str]] = [] + for representative, titles in representative_titles.items(): + best_score = 0 + for candidate_title in titles: + if not candidate_title: + continue + if title == candidate_title: + best_score = max(best_score, 100) + elif len(title) >= 8 and len(candidate_title) >= 8 and (title in candidate_title or candidate_title in title): + best_score = max(best_score, min(len(title), len(candidate_title))) + if best_score >= 8: + matches.append((best_score, representative)) + if not matches: + continue + matches.sort(reverse=True) + if len(matches) > 1 and matches[0][0] == matches[1][0]: + continue + inferred[normalized_code] = matches[0][1] + return inferred + + def _cost_analysis_latest_row(rows: list[dict[str, Any]]) -> dict[str, Any]: def sort_key(row: dict[str, Any]) -> tuple[str, str]: code = normalize_text(row.get("support_dept_code")).upper() @@ -12992,6 +14159,9 @@ def _cost_analysis_aggregate_rows( aggregate["contract_amount"] = sum(normalize_amount(row.get("contract_amount")) for row in group_rows) aggregate["billing_amount"] = sum(normalize_amount(row.get("billing_amount")) for row in group_rows) aggregate["collection_amount"] = sum(normalize_amount(row.get("collection_amount")) for row in group_rows) + aggregate["period_billing_amount"] = sum(normalize_amount(row.get("period_billing_amount")) for row in group_rows) + aggregate["period_collection_amount"] = sum(normalize_amount(row.get("period_collection_amount")) for row in group_rows) + aggregate["period_revenue_amount"] = sum(normalize_amount(row.get("period_revenue_amount") or row.get("revenue_amount")) for row in group_rows) aggregate["contract_balance_amount"] = 0.0 aggregate["revenue_amount"] = sum(normalize_amount(row.get("revenue_amount")) for row in group_rows) aggregate["phases"] = _cost_analysis_empty_phase_totals() @@ -13009,6 +14179,11 @@ def _cost_analysis_aggregate_rows( for item_key, amount in buckets.items(): if item_key in aggregate["allocated"][phase]: aggregate["allocated"][phase][item_key] += normalize_amount(amount) + aggregate["period_cost_total"] = sum(normalize_amount(row.get("period_cost_total") or row.get("cost_total")) for row in group_rows) + aggregate["period_sga_total"] = sum(normalize_amount(row.get("period_sga_total") or row.get("sga_total")) for row in group_rows) + aggregate["period_sales_total"] = sum(normalize_amount(row.get("period_sales_total") or row.get("sales_total")) for row in group_rows) + aggregate["period_total_cost"] = sum(normalize_amount(row.get("period_total_cost") or row.get("total_cost")) for row in group_rows) + aggregate["period_profit_amount"] = aggregate["period_revenue_amount"] - aggregate["period_total_cost"] _cost_analysis_finalize_row(aggregate) result.append(aggregate) return result @@ -13100,8 +14275,17 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: continue visible_candidate_codes.add(x_owner_map.get(source_code, xyz_code_map.get(source_code, source_code))) - period_hanmac_hours = _cost_analysis_load_hanmac_project_hours(start_date, end_date, project_meta) - visible_candidate_codes.update(period_hanmac_hours.keys()) + annual_hanmac_project_hours_by_year, annual_hanmac_labor_by_year = _cost_analysis_load_hanmac_hours_and_labor_yearly( + start_date, + end_date, + project_meta, + None, + ) + visible_candidate_codes.update( + code + for code_map in annual_hanmac_project_hours_by_year.values() + for code in code_map + ) source_candidate_codes: set[str] = set(visible_candidate_codes) for target_code in list(visible_candidate_codes): @@ -13113,12 +14297,15 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: if target_code in visible_candidate_codes: source_candidate_codes.add(source_code) - hanmac_labor_map = _cost_analysis_load_hanmac_labor_map_yearly( - start_date, - end_date, - project_meta, - visible_candidate_codes or None, - ) + hanmac_labor_map: dict[str, dict[str, float]] = {} + for code_map in annual_hanmac_labor_by_year.values(): + for code, phase_amounts in code_map.items(): + if visible_candidate_codes and code not in visible_candidate_codes: + continue + target = hanmac_labor_map.setdefault(code, {"pre": 0.0, "during": 0.0, "post": 0.0}) + for phase, amount in phase_amounts.items(): + if phase in target: + target[phase] += normalize_amount(amount) def ensure_row(code: str) -> dict[str, Any]: normalized_code = normalize_text(code).upper() @@ -13274,6 +14461,7 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: amount = normalize_amount(row.get("amount")) if bucket == "revenue": report_row["revenue_amount"] += amount + report_row["period_revenue_amount"] += amount continue if bucket not in {"cost", "sga"}: continue @@ -13302,6 +14490,7 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: if code != target_code and code not in report_row["direct_codes"]: report_row["direct_codes"].append(code) report_row["collection_amount"] += normalize_amount(collection_row.get("collection_amount")) + report_row["period_collection_amount"] += normalize_amount(collection_row.get("period_collection_amount")) if normalize_amount(collection_row.get("period_collection_amount")): visible_activity_codes.add(target_code) @@ -13318,6 +14507,7 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: if code != target_code and code not in report_row["direct_codes"]: report_row["direct_codes"].append(code) report_row["collection_amount"] += amount + report_row["period_collection_amount"] += normalize_amount(collection_row.get("period_collection_amount")) if normalize_amount(collection_row.get("period_collection_amount")): visible_activity_codes.add(target_code) @@ -13333,30 +14523,11 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: if code != target_code and code not in report_row["direct_codes"]: report_row["direct_codes"].append(code) report_row["billing_amount"] += amount + report_row["period_billing_amount"] += normalize_amount(billing_row.get("period_billing_amount")) if normalize_amount(billing_row.get("period_billing_amount")): visible_activity_codes.add(target_code) annual_sga_totals = {int(row["posting_year"]): normalize_amount(row.get("amount")) for row in annual_sga_rows if row.get("posting_year")} - annual_hanmac_project_hours_by_year: dict[int, dict[str, dict[str, float]]] = {} - annual_prefix_metric = _cost_analysis_select_hanmac_prefix_metric(start_date, end_date) - annual_prefix_end = _parse_iso_date((annual_prefix_metric or {}).get("end_date")) - annual_loop_start = start_date - if annual_prefix_metric and annual_prefix_end and annual_prefix_end >= start_date: - annual_hanmac_project_hours_by_year.update(_cost_analysis_load_hanmac_project_hours_by_year( - start_date, - min(annual_prefix_end, end_date), - project_meta, - visible_candidate_codes or None, - )) - annual_loop_start = min(annual_prefix_end, end_date) + timedelta(days=1) - if annual_loop_start <= end_date: - for year_slice in _iter_year_slices(annual_loop_start, end_date): - annual_hanmac_project_hours_by_year.update(_cost_analysis_load_hanmac_project_hours_by_year( - year_slice["start"], - year_slice["end"], - project_meta, - visible_candidate_codes or None, - )) for year_slice in _iter_year_slices(start_date, end_date): year = int(year_slice["year"]) annual_sga_total = annual_sga_totals.get(year, 0.0) @@ -13420,6 +14591,14 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: "contract_amount": sum(normalize_amount(row.get("contract_amount")) for row in final_rows), "billing_amount": sum(normalize_amount(row.get("billing_amount")) for row in final_rows), "collection_amount": sum(normalize_amount(row.get("collection_amount")) for row in final_rows), + "period_billing_amount": sum(normalize_amount(row.get("period_billing_amount")) for row in final_rows), + "period_collection_amount": sum(normalize_amount(row.get("period_collection_amount")) for row in final_rows), + "period_revenue_amount": sum(normalize_amount(row.get("period_revenue_amount") or row.get("revenue_amount")) for row in final_rows), + "period_cost_total": sum(normalize_amount(row.get("period_cost_total") or row.get("cost_total")) for row in final_rows), + "period_sga_total": sum(normalize_amount(row.get("period_sga_total") or row.get("sga_total")) for row in final_rows), + "period_sales_total": sum(normalize_amount(row.get("period_sales_total") or row.get("sales_total")) for row in final_rows), + "period_total_cost": sum(normalize_amount(row.get("period_total_cost") or row.get("total_cost")) for row in final_rows), + "period_profit_amount": sum(normalize_amount(row.get("period_profit_amount") or row.get("profit_amount")) for row in final_rows), "contract_balance_amount": sum(normalize_amount(row.get("contract_balance_amount")) for row in final_rows), "revenue_amount": sum(normalize_amount(row.get("revenue_amount")) for row in final_rows), "cost_total": sum(normalize_amount(row.get("cost_total")) for row in final_rows), @@ -13433,6 +14612,9 @@ def _cost_analysis_build_payload(start_date_text: str, end_date_text: str, mode: summary["contract_profit_rate"] = _safe_ratio(summary["profit_amount"], summary["contract_amount"]) summary["revenue_profit_rate"] = _safe_ratio(summary["profit_amount"], summary["revenue_amount"]) summary["collection_profit_rate"] = _safe_ratio(summary["profit_amount"], summary["collection_amount"]) + summary["period_revenue_profit_rate"] = _safe_ratio(summary["period_profit_amount"], summary["period_revenue_amount"]) + summary["period_collection_profit_rate"] = _safe_ratio(summary["period_profit_amount"], summary["period_collection_amount"]) + summary["cumulative_profit_rate"] = _safe_ratio(summary["collection_amount"] - summary["total_cost"], summary["collection_amount"]) payload = { "start_date": start_date.isoformat(), "end_date": end_date.isoformat(), @@ -13537,6 +14719,42 @@ def render_wehago_compare_page( return templates.TemplateResponse(request, "wehago_compare.html", context) +def render_wehago_benefit_entertainment_page( + request: Request, + start_year: int = WEHAGO_BENEFIT_DEFAULT_START_YEAR, + end_year: int = WEHAGO_BENEFIT_DEFAULT_END_YEAR, + account_group: str = "all", + category: str = "all", + person_keyword: str = "", + desc_keyword: str = "", + include_adjustments: bool = False, + message: str = "", +) -> HTMLResponse: + init_db() + report = get_wehago_benefit_entertainment_report( + start_year=start_year, + end_year=end_year, + account_group=account_group, + category=category, + person_keyword=person_keyword, + desc_keyword=desc_keyword, + include_adjustments=include_adjustments, + limit=300, + ) + context = { + **base_context(request, message), + "benefit_report": report, + "benefit_report_json": jsonable_encoder( + { + "detail_rows": report.get("all_detail_rows", []), + "vendor_summary_rows": report.get("vendor_summary_rows", []), + "category_options": report.get("category_options", []), + } + ), + } + return templates.TemplateResponse(request, "wehago_benefit_entertainment.html", context) + + def render_hanmac_browser_page( request: Request, message: str = "", @@ -19672,6 +20890,86 @@ async def wehago_compare(request: Request, start_year: str | None = None, end_ye return HTMLResponse("

서버 오류

로그를 확인해주세요.

", status_code=500) +@app.get("/wehago-benefit-entertainment") +async def wehago_benefit_entertainment( + request: Request, + start_year: str | None = None, + end_year: str | None = None, + account_group: str = "all", + category: str = "all", + person_keyword: str = "", + desc_keyword: str = "", + include_adjustments: str | None = None, +): + try: + return await run_in_threadpool( + render_wehago_benefit_entertainment_page, + request, + start_year=_normalize_report_year(start_year, WEHAGO_BENEFIT_DEFAULT_START_YEAR), + end_year=_normalize_report_year(end_year, WEHAGO_BENEFIT_DEFAULT_END_YEAR), + account_group=account_group, + category=category, + person_keyword=person_keyword, + desc_keyword=desc_keyword, + include_adjustments=_normalize_report_bool(include_adjustments), + ) + except Exception as exc: + logger.exception("복리/접대비 보고서 페이지 에러: %s", exc) + return HTMLResponse("

서버 오류

로그를 확인해주세요.

", status_code=500) + + +@app.get("/wehago-benefit-entertainment/export") +async def wehago_benefit_entertainment_export( + start_year: str | None = None, + end_year: str | None = None, + account_group: str = "all", + category: str = "all", + person_keyword: str = "", + desc_keyword: str = "", + include_adjustments: str | None = None, +): + try: + report = await run_in_threadpool( + get_wehago_benefit_entertainment_report, + start_year=_normalize_report_year(start_year, WEHAGO_BENEFIT_DEFAULT_START_YEAR), + end_year=_normalize_report_year(end_year, WEHAGO_BENEFIT_DEFAULT_END_YEAR), + account_group=account_group, + category=category, + person_keyword=person_keyword, + desc_keyword=desc_keyword, + include_adjustments=_normalize_report_bool(include_adjustments), + limit=None, + ) + file_name, content = await run_in_threadpool(export_wehago_benefit_entertainment_xlsx, report) + return Response( + content=content, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": f'attachment; filename="{file_name}"'}, + ) + except Exception as exc: + logger.exception("복리/접대비 엑셀 다운로드 에러: %s", exc) + return JSONResponse(content={"error": str(exc)}, status_code=500) + + +@app.post("/wehago-benefit-entertainment/api/category") +async def wehago_benefit_entertainment_save_category(request: Request): + try: + payload = await request.json() + if not isinstance(payload, dict): + payload = {} + result = await run_in_threadpool( + save_wehago_benefit_category_override, + payload.get("ledger_row_id"), + payload.get("category"), + ) + return JSONResponse(content=jsonable_encoder(result)) + except ValueError as exc: + return JSONResponse(content={"ok": False, "error": str(exc)}, status_code=400) + except Exception as exc: + logger.exception("복리/접대비 분류 저장 에러: %s", exc) + return JSONResponse(content={"ok": False, "error": str(exc)}, status_code=500) + + @app.post("/wehago-compare/upload-erp") async def upload_wehago_erp_file( request: Request, diff --git a/scripts/prune_wehago_projection_history.py b/scripts/prune_wehago_projection_history.py new file mode 100644 index 0000000..08997f5 --- /dev/null +++ b/scripts/prune_wehago_projection_history.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import argparse +import json +import sqlite3 +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from runtime_config import DB_PATH + + +DERIVED_SIGNATURE_TABLES = ( + ("wehago_compare_query_groups", "signature"), + ("wehago_compare_query_rows", "signature"), + ("wehago_compare_query_metrics", "signature"), + ("wehago_compare_query_page_cache", "signature"), + ("wehago_compare_final_status_projection", "signature"), + ("wehago_metric_count_cache", "signature"), + ("wehago_summary_range_cache", "signature"), +) + + +def active_signature(conn: sqlite3.Connection, start_year: int, end_year: int) -> str: + row = conn.execute( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = ? + LIMIT 1 + """, + (f"wehago_active_query_projection:{start_year}:{end_year}",), + ).fetchone() + if not row: + return "" + try: + payload = json.loads(row[0] or "{}") + except Exception: + return "" + return str(payload.get("signature") or "").strip() if isinstance(payload, dict) else "" + + +def table_exists(conn: sqlite3.Connection, table_name: str) -> bool: + return bool( + conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1", + (table_name,), + ).fetchone() + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prune obsolete WEHAGO derived projection records.") + parser.add_argument("--start-year", type=int, default=2025) + parser.add_argument("--end-year", type=int, default=2025) + parser.add_argument("--execute", action="store_true", help="Actually delete rows. Without this flag, only prints counts.") + args = parser.parse_args() + + conn = sqlite3.connect(DB_PATH) + signature = active_signature(conn, args.start_year, args.end_year) + if not signature: + raise SystemExit("No active projection signature was found. Refusing to prune.") + + results: list[dict[str, object]] = [] + conn.execute("BEGIN") + try: + for table_name, signature_column in DERIVED_SIGNATURE_TABLES: + if not table_exists(conn, table_name): + continue + count = int( + conn.execute( + f""" + SELECT COUNT(*) + FROM {table_name} + WHERE start_year = ? AND end_year = ? + AND {signature_column} <> ? + """, + (args.start_year, args.end_year, signature), + ).fetchone()[0] + or 0 + ) + results.append({"table": table_name, "obsolete_rows": count}) + if args.execute and count: + conn.execute( + f""" + DELETE FROM {table_name} + WHERE start_year = ? AND end_year = ? + AND {signature_column} <> ? + """, + (args.start_year, args.end_year, signature), + ) + if args.execute: + conn.commit() + else: + conn.rollback() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + print( + json.dumps( + { + "start_year": args.start_year, + "end_year": args.end_year, + "active_signature": signature, + "execute": bool(args.execute), + "tables": results, + "obsolete_total": sum(int(row["obsolete_rows"]) for row in results), + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/rebuild_compare_ranges.py b/scripts/rebuild_compare_ranges.py index b2dba5b..6a892f8 100644 --- a/scripts/rebuild_compare_ranges.py +++ b/scripts/rebuild_compare_ranges.py @@ -11,6 +11,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from main import engine from wehago_compare import ( _build_db_state_signature, + _ensure_year_export_row_cache, _get_fast_year_export_row_cache_signature, _load_snapshot_status_map, _project_year_export_row_cache_from_latest_resolved, @@ -52,7 +53,18 @@ def main() -> None: projected = _project_year_export_row_cache_from_latest_resolved(conn, year, signature) if not projected: print({"step": "fast_year_projection_miss", "year": year}, flush=True) - _refresh_year_resolved_sections(conn, year) + selected_signature = _ensure_year_export_row_cache(conn, year) + else: + selected_signature = _get_fast_year_export_row_cache_signature(conn, year) + if selected_signature: + _upsert_snapshot_status( + conn, + year, + signature=selected_signature, + state="ready", + row_counts={}, + built_now=True, + ) else: _upsert_snapshot_status( conn, @@ -62,7 +74,6 @@ def main() -> None: row_counts={}, built_now=True, ) - selected_signature = _get_fast_year_export_row_cache_signature(conn, year) print( { "step": "fast_year_projection_done", diff --git a/scripts/reconcile_wehago_projection_to_db.py b/scripts/reconcile_wehago_projection_to_db.py index 4333b8f..eee3a15 100644 --- a/scripts/reconcile_wehago_projection_to_db.py +++ b/scripts/reconcile_wehago_projection_to_db.py @@ -16,13 +16,19 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from runtime_config import DB_PATH from wehago_compare import ( QUERY_PROJECTION_VERSION, + _account_nature_signature, _account_category_pair_allowed, _classify_account_category, _classify_account_family, _is_wehago_excepted_voucher_group, _is_vat_family, + _group_has_offset_tax_invoice_structure, + _group_has_tax_invoice_cancel_signal, + _offset_group_vector, _move_wehago_confirmed_reversal_pairs_to_excepted, - _move_wehago_offset_tax_invoice_groups_to_excepted, + _voucher_group_has_review_reason, + _voucher_group_month_days, + _voucher_groups_within_days, _nature_compatible, build_voucher_row_key, clean, @@ -30,11 +36,27 @@ from wehago_compare import ( YEAR = 2025 +RECONCILED_PROJECTION_VERSION = "db-reconciled-v2" WEHAGO_STATUSES = ("voucher_matched", "voucher_unmatched", "voucher_recheck", "voucher_excepted") ERP_STATUSES = ("erp_voucher_matched", "erp_voucher_unmatched") ALL_VOUCHER_STATUSES = WEHAGO_STATUSES + ERP_STATUSES RAW_ERP_ROWS_BY_DRAFT_BASE: dict[str, list[dict[str, Any]]] = defaultdict(list) MANUAL_OFFSET_EXCEPTED_IDENTITIES: set[str] = set() +EXCEPTED_REASON_TOKENS = ( + "WEHAGO_EXCEPTED_OFFSET_REVERSAL_PAIR", + "WEHAGO_EXCEPTED_CONFIRMED_REVERSAL_PAIR", + "WEHAGO_EXCEPTED_OFFSET_ENTRY", + "WEHAGO_EXCEPTED_SUBSTITUTION_ENTRY", + "WEHAGO_EXCEPTED_AUDIT_ADJUSTMENT", + "WEHAGO_EXCEPTED_CLOSING_REVERSAL", + "WEHAGO_EXCEPTED_OPENING_BALANCE", + "WEHAGO_EXCEPTED_YEAR_OPENING_SUBSTITUTION", + "WEHAGO_EXCEPTED_EXACT_REVERSAL_PAIR", + "WEHAGO_EXCEPTED_MANAGEMENT_ITEM_REVERSAL_PAIR", + "WEHAGO_EXCEPTED_CANCEL_REISSUE_CANCEL", + "WEHAGO_EXCEPTED_CANCEL_REISSUE_SUPERSEDED", + "MANUAL_OFFSET_PAIR_EXCEPTED", +) GROUP_COLUMNS = ( "start_year", @@ -140,6 +162,10 @@ def dict_row(row: sqlite3.Row) -> dict[str, Any]: return {key: row[key] for key in row.keys()} +def log_step(message: str) -> None: + print(f"[reconcile] {datetime.now().isoformat(timespec='seconds')} {message}", flush=True) + + def latest_projection_signature(cur: sqlite3.Cursor) -> str: row = cur.execute( """ @@ -154,7 +180,7 @@ def latest_projection_signature(cur: sqlite3.Cursor) -> str: try: payload = json.loads(row[0] or "{}") signature = clean(payload.get("signature")) - if signature and "|db-reconciled-v1|" not in signature: + if signature and "|db-reconciled-" not in signature: exists = cur.execute( """ SELECT 1 @@ -173,7 +199,7 @@ def latest_projection_signature(cur: sqlite3.Cursor) -> str: SELECT signature, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature LIKE ? - AND signature NOT LIKE '%|db-reconciled-v1|%' + AND signature NOT LIKE '%|db-reconciled-%' GROUP BY signature ORDER BY max_updated_at DESC LIMIT 1 @@ -186,13 +212,41 @@ def latest_projection_signature(cur: sqlite3.Cursor) -> str: SELECT signature, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? - AND signature NOT LIKE '%|db-reconciled-v1|%' + AND signature NOT LIKE '%|db-reconciled-%' GROUP BY signature ORDER BY max_updated_at DESC LIMIT 1 """, (YEAR, YEAR), ).fetchone() + if not row: + settings_row = cur.execute( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = ? + LIMIT 1 + """, + (f"wehago_active_query_projection:{YEAR}:{YEAR}",), + ).fetchone() + if settings_row: + try: + payload = json.loads(settings_row[0] or "{}") + signature = clean(payload.get("signature")) if isinstance(payload, dict) else "" + if signature: + exists = cur.execute( + """ + SELECT 1 + FROM wehago_compare_query_groups + WHERE start_year = ? AND end_year = ? AND signature = ? + LIMIT 1 + """, + (YEAR, YEAR, signature), + ).fetchone() + if exists: + return signature + except Exception: + pass if not row: raise RuntimeError("No current query projection was found.") return clean(row["signature"]) @@ -439,6 +493,112 @@ def direct_row_matches_account_nature(row: dict[str, Any]) -> bool: ) +def compact_account_name(value: Any) -> str: + return normalized_compact(re.sub(r"^\d+\s*", "", clean(value))) + + +def row_account_category(row: dict[str, Any], prefix: str) -> str: + account_name = clean(row.get(f"{prefix}_account_name")) + compact = compact_account_name(account_name) + if any(marker in compact for marker in ("세금과공과", "퇴직금", "퇴직급여", "임금", "제수당", "보험료", "보증수수료")): + return "expense" + return _classify_account_category("", account_name) + + +def row_account_family(row: dict[str, Any], prefix: str) -> str: + return _classify_account_family("", row.get(f"{prefix}_account_name")) + + +def is_business_category(category: str) -> bool: + return category in {"expense", "revenue"} + + +def is_vat_row(row: dict[str, Any], prefix: str) -> bool: + return _is_vat_family(row_account_family(row, prefix)) + + +def is_business_row(row: dict[str, Any], prefix: str) -> bool: + family = row_account_family(row, prefix) + if _is_vat_family(family) or family in {"bank", "payable", "receivable", "advance"}: + return False + return is_business_category(row_account_category(row, prefix)) + + +def is_settlement_row(row: dict[str, Any], prefix: str) -> bool: + family = row_account_family(row, prefix) + category = row_account_category(row, prefix) + return family in {"bank", "payable", "receivable", "advance"} or category in {"asset", "liability"} + + +def row_principle_signature(row: dict[str, Any], prefix: str) -> str: + return _account_nature_signature("", row.get(f"{prefix}_account_name"), effective_account_side(row, prefix)) + + +def row_business_match_key(row: dict[str, Any], prefix: str) -> tuple[str, str, str, str]: + return ( + row_account_category(row, prefix), + compact_account_name(row.get(f"{prefix}_account_name")), + amount_key(row_side_amount(row, prefix)), + normalized_compact(row.get(f"{prefix}_desc")), + ) + + +def row_principle_pair_allowed(ledger_row: dict[str, Any], voucher_row: dict[str, Any]) -> bool: + ledger_category = row_account_category(ledger_row, "ledger") + voucher_category = row_account_category(voucher_row, "voucher") + ledger_family = row_account_family(ledger_row, "ledger") + voucher_family = row_account_family(voucher_row, "voucher") + if _is_vat_family(ledger_family) or _is_vat_family(voucher_family): + return _is_vat_family(ledger_family) and _is_vat_family(voucher_family) + if ledger_category != voucher_category or ledger_category not in {"expense", "revenue"}: + return False + ledger_account = compact_account_name(ledger_row.get("ledger_account_name")) + voucher_account = compact_account_name(voucher_row.get("voucher_account_name")) + if ( + ledger_category == "expense" + and effective_account_side(ledger_row, "ledger") == effective_account_side(voucher_row, "voucher") + and ( + ("보증수수료" in ledger_account and ("보험료" in voucher_account or "보증보험" in voucher_account)) + or ("보증수수료" in voucher_account and ("보험료" in ledger_account or "보증보험" in ledger_account)) + ) + ): + return True + return _nature_compatible( + "", + ledger_row.get("ledger_account_name"), + effective_account_side(ledger_row, "ledger"), + "", + voucher_row.get("voucher_account_name"), + effective_account_side(voucher_row, "voucher"), + ) + + +def group_has_wehago_rows(group: dict[str, Any]) -> bool: + return any(has_wehago_value(row) for row in group.get("rows") or []) + + +def group_has_real_erp_candidate(group: dict[str, Any]) -> bool: + return any(has_erp_value(row) for row in group.get("rows") or []) + + +def group_has_unmatched_wehago_rows(group: dict[str, Any]) -> bool: + return any(has_wehago_value(row) and not has_erp_value(row) for row in group.get("rows") or []) + + +def group_review_text(group: dict[str, Any]) -> str: + return " ".join( + [ + clean((group.get("summary") or {}).get("review_reason")), + *(clean(row.get("review_reason")) for row in group.get("rows") or []), + ] + ) + + +def group_has_excepted_reason(group: dict[str, Any]) -> bool: + review_text = group_review_text(group) + return any(token in review_text for token in EXCEPTED_REASON_TOKENS) + + def row_side_amount(row: dict[str, Any], prefix: str) -> float: return max( abs(parse_amount(row.get(f"{prefix}_debit"))), @@ -465,6 +625,25 @@ def raw_erp_entry_amount_side(entry: dict[str, Any], category: str) -> tuple[flo return 0.0, "either" +def raw_erp_entry_signed_amount_side(entry: dict[str, Any], category: str) -> tuple[float, str]: + fields = ( + ("debit_supply", "debit"), + ("credit_supply", "credit"), + ) + if category in {"asset", "liability"}: + fields = ( + ("debit_supply", "debit"), + ("credit_supply", "credit"), + ("debit_tax", "debit"), + ("credit_tax", "credit"), + ) + for field, side in fields: + amount = parse_amount(entry.get(field)) + if abs(amount) >= 0.5: + return amount, side + return 0.0, "either" + + def raw_erp_entry_to_row(entry: dict[str, Any], ledger_row: dict[str, Any], side: str, reason: str) -> dict[str, Any]: amount, _entry_side = raw_erp_entry_amount_side( entry, @@ -501,6 +680,167 @@ def raw_erp_entry_to_row(entry: dict[str, Any], ledger_row: dict[str, Any], side return row +def normalized_compact(value: Any) -> str: + return re.sub(r"\s+", "", clean(value)).lower() + + +def row_draft_bases(rows: list[dict[str, Any]], summary: dict[str, Any]) -> set[str]: + bases: set[str] = set() + for value in [summary.get("draft_no"), *(row.get("draft_no") for row in rows)]: + for part in re.split(r"[,/]\s*", clean(value)): + base = erp_voucher_base(part) + if base: + bases.add(base) + return bases + + +def dates_match_wehago(proof_date: Any, ledger_date: Any, fiscal_year: int) -> bool: + proof = clean(proof_date) + ledger = clean(ledger_date) + if not proof or not ledger: + return False + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", proof): + proof_key = proof + else: + matched = re.search(r"(20\d{2})[-./]?(\d{1,2})[-./]?(\d{1,2})", proof) + if not matched: + return False + proof_key = f"{int(matched.group(1)):04d}-{int(matched.group(2)):02d}-{int(matched.group(3)):02d}" + if re.fullmatch(r"\d{1,2}[-./]\d{1,2}", ledger): + month, day = re.split(r"[-./]", ledger) + ledger_key = f"{int(fiscal_year):04d}-{int(month):02d}-{int(day):02d}" + elif re.fullmatch(r"\d{4}-\d{2}-\d{2}", ledger): + ledger_key = ledger + else: + return False + return proof_key == ledger_key + + +def revenue_entry_semantically_matches(row: dict[str, Any], entry: dict[str, Any]) -> bool: + ledger_account = normalized_compact(row.get("ledger_account_name")) + ledger_desc = normalized_compact(row.get("ledger_desc")) + entry_text = normalized_compact(" ".join([clean(entry.get("account_name")), clean(entry.get("desc1")), clean(entry.get("desc2"))])) + if "주차" in ledger_account: + return "주차" in entry_text + if "임대" in ledger_account: + return ("임대" in entry_text or "관리비" in entry_text) and "주차" not in entry_text + if "관리" in ledger_account: + return "관리" in entry_text + ledger_tokens = {token for token in re.split(r"[^0-9a-z가-힣]+", ledger_account + " " + ledger_desc) if len(token) >= 2} + entry_tokens = {token for token in re.split(r"[^0-9a-z가-힣]+", entry_text) if len(token) >= 2} + return bool(ledger_tokens & entry_tokens) + + +def find_amount_matching_subset( + entries: list[dict[str, Any]], + target_amount: float, + category: str, +) -> tuple[list[dict[str, Any]], str] | None: + candidates: list[tuple[dict[str, Any], float, str]] = [] + for entry in entries[:12]: + amount, side = raw_erp_entry_amount_side(entry, category) + if amount > 0: + candidates.append((entry, amount, side)) + best: tuple[list[dict[str, Any]], str] | None = None + for mask in range(1, 1 << len(candidates)): + selected: list[dict[str, Any]] = [] + total = 0.0 + side = "" + for index, (entry, amount, entry_side) in enumerate(candidates): + if not (mask & (1 << index)): + continue + if side and entry_side != side: + selected = [] + break + side = entry_side + selected.append(entry) + total += amount + if not selected or len(selected) < 2: + continue + if abs(total - target_amount) < 0.5: + if best is None or len(selected) < len(best[0]): + best = (selected, side) + return best + + +def retarget_to_same_draft_split_revenue( + row: dict[str, Any], + group_rows: list[dict[str, Any]], + summary: dict[str, Any], +) -> dict[str, Any] | None: + if not has_wehago_value(row) or has_erp_value(row): + return None + ledger_category = _classify_account_category("", row.get("ledger_account_name")) + if ledger_category != "revenue": + return None + ledger_side = effective_account_side(row, "ledger") + ledger_amount = row_side_amount(row, "ledger") + if ledger_side not in {"debit", "credit"} or ledger_amount <= 0: + return None + fiscal_year = int(row.get("fiscal_year") or summary.get("fiscal_year") or YEAR) + bases = row_draft_bases(group_rows, summary) + for base in sorted(bases): + entries = [ + entry + for entry in RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, []) + if _classify_account_category(entry.get("account_code"), entry.get("account_name")) == "revenue" + and dates_match_wehago(entry.get("proof_date"), row.get("ledger_date") or summary.get("ledger_date"), fiscal_year) + and revenue_entry_semantically_matches(row, entry) + and _nature_compatible("", row.get("ledger_account_name"), ledger_side, entry.get("account_code"), entry.get("account_name"), ledger_side) + and ( + not clean(row.get("ledger_vendor")) + or not clean(entry.get("vendor_name")) + or normalized_compact(row.get("ledger_vendor")) in normalized_compact(entry.get("vendor_name")) + or normalized_compact(entry.get("vendor_name")) in normalized_compact(row.get("ledger_vendor")) + ) + ] + matched = find_amount_matching_subset(entries, ledger_amount, "revenue") + if matched is None: + continue + selected, side = matched + if side != ledger_side: + continue + payload = dict(row) + payload["proof_date"] = clean(selected[0].get("proof_date")) + payload["draft_no"] = ", ".join(clean(entry.get("draft_no")) or clean(entry.get("confirmed_no")) for entry in selected if clean(entry.get("draft_no")) or clean(entry.get("confirmed_no"))) + payload["voucher_account_name"] = ", ".join(clean(entry.get("account_name")) for entry in selected if clean(entry.get("account_name"))) + payload["voucher_vendor"] = clean(selected[0].get("vendor_name")) + payload["voucher_debit"] = ledger_amount if side == "debit" else 0 + payload["voucher_credit"] = ledger_amount if side == "credit" else 0 + payload["voucher_desc"] = " / ".join( + clean(" ".join(part for part in (entry.get("desc1"), entry.get("desc2")) if clean(part))) + for entry in selected + if clean(entry.get("desc1")) or clean(entry.get("desc2")) + ) + payload["status_label"] = "Matched" + payload["review_reason"] = "PROJECTION_RECONCILE_SPLIT_REVENUE_SAME_DRAFT_PROOF_DATE" + payload["voucher_row_key"] = build_voucher_row_key(payload) + payload["match_identity_key"] = "|".join( + clean(part) + for part in ( + payload.get("fiscal_year"), + payload.get("voucher_no"), + payload.get("ledger_row_key"), + payload.get("draft_no"), + payload.get("voucher_row_key"), + ) + if clean(part) + ) + return payload + return None + + +def apply_same_draft_split_revenue_matches( + rows: list[dict[str, Any]], + summary: dict[str, Any], +) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for row in rows: + retargeted = retarget_to_same_draft_split_revenue(row, rows, summary) + result.append(retargeted if retargeted is not None else row) + return result + + def retarget_to_same_draft_business_account(row: dict[str, Any]) -> dict[str, Any] | None: if not (has_wehago_value(row) and clean(row.get("draft_no"))): return None @@ -619,6 +959,7 @@ def allowed_erp_drafts_for_group(rows: list[dict[str, Any]]) -> set[str]: def clean_group_rows(group: dict[str, Any], status_key: str) -> dict[str, Any]: rows = [dict(row) for row in group.get("rows") or []] + summary = dict(group.get("summary") or {}) if status_key == "voucher_matched": normalized_rows: list[dict[str, Any]] = [] for row in rows: @@ -631,6 +972,7 @@ def clean_group_rows(group: dict[str, Any], status_key: str) -> dict[str, Any]: else: normalized_rows.append(blank_erp_side(row, "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE")) rows = normalized_rows + rows = apply_same_draft_split_revenue_matches(rows, summary) allowed_drafts = allowed_erp_drafts_for_group(rows) cleaned: list[dict[str, Any]] = [] for row in rows: @@ -662,7 +1004,7 @@ def clean_group_rows(group: dict[str, Any], status_key: str) -> dict[str, Any]: cleaned.append(blank_erp_side(row, "PROJECTION_RECONCILE_RECHECK_NO_REAL_ERP_CANDIDATE")) continue cleaned.append(row) - rows = cleaned + rows = apply_same_draft_split_revenue_matches(cleaned, summary) seen_ledger: set[tuple[Any, ...]] = set() seen_voucher: set[tuple[Any, ...]] = set() @@ -737,33 +1079,31 @@ def split_group_by_wehago_voucher(group: dict[str, Any], status_key: str) -> dic def normalized_wehago_status(status_key: str, group: dict[str, Any]) -> str: - if status_key == "voucher_matched": - if any( - has_wehago_value(row) and has_erp_value(row) - for row in group.get("rows") or [] - ): - return "voucher_matched" - review_text = " ".join( - clean(row.get("review_reason")) - for row in group.get("rows") or [] - ) - if any( - token in review_text - for token in ( - "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE", - "PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP", - ) - ): - return "voucher_recheck" + if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES: + return "voucher_excepted" + if group_has_excepted_reason(group): + return "voucher_excepted" + is_excepted, _reason = _is_wehago_excepted_voucher_group(group) + if is_excepted: + return "voucher_excepted" + has_wehago = group_has_wehago_rows(group) + has_real_erp_candidate = group_has_real_erp_candidate(group) + if not has_wehago: return "voucher_unmatched" + review_text = group_review_text(group) + if status_key == "voucher_matched": + recheck_tokens = ( + "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE", + "PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP", + "PROJECTION_RECONCILE_PARTIAL_MATCH_WEHAGO_ROW_UNMATCHED", + ) + if has_real_erp_candidate and not any(token in review_text for token in recheck_tokens): + return "voucher_matched" + return "voucher_recheck" if has_real_erp_candidate else "voucher_unmatched" + if status_key in {"voucher_unmatched", "voucher_excepted"}: + return "voucher_recheck" if has_real_erp_candidate else "voucher_unmatched" if status_key != "voucher_recheck": return status_key - review_text = " ".join( - [ - clean((group.get("summary") or {}).get("review_reason")), - *(clean(row.get("review_reason")) for row in group.get("rows") or []), - ] - ) if any( reason in review_text for reason in ( @@ -773,7 +1113,12 @@ def normalized_wehago_status(status_key: str, group: dict[str, Any]) -> str: ) ): return "voucher_recheck" - has_real_erp_candidate = any(has_erp_value(row) for row in group.get("rows") or []) + if ( + "PROJECTION_RECONCILE_SPLIT_REVENUE_SAME_DRAFT_PROOF_DATE" in review_text + and has_real_erp_candidate + and not group_has_unmatched_wehago_rows(group) + ): + return "voucher_matched" return "voucher_recheck" if has_real_erp_candidate else "voucher_unmatched" @@ -796,13 +1141,11 @@ def mark_excepted(group: dict[str, Any], reason: str) -> dict[str, Any]: payload = {"summary": summary, "rows": rows} payload["summary"]["status_label"] = "Excepted" existing_reason = clean(payload["summary"].get("review_reason")) - if reason not in existing_reason: - payload["summary"]["review_reason"] = " / ".join(item for item in (existing_reason, reason) if item) + payload["summary"]["review_reason"] = dedup_review_reason_text(existing_reason, reason) for row in payload["rows"]: row["status_label"] = "Excepted" row_reason = clean(row.get("review_reason")) - if reason not in row_reason: - row["review_reason"] = " / ".join(item for item in (row_reason, reason) if item) + row["review_reason"] = dedup_review_reason_text(row_reason, reason) payload["summary"] = rebuild_summary(payload, "voucher_excepted") return payload @@ -929,8 +1272,7 @@ def apply_net_adjustment_component_matches(status_groups: dict[str, list[dict[st def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None: excepted: list[dict[str, Any]] = [] - recheck_from_matched: list[dict[str, Any]] = [] - unmatched_from_excepted: list[dict[str, Any]] = [] + relaxed_from_excepted: dict[str, list[dict[str, Any]]] = defaultdict(list) for group in list(status_groups.get("voucher_excepted") or []): existing_reason = clean((group.get("summary") or {}).get("review_reason")) if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES: @@ -943,8 +1285,9 @@ def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None if is_excepted: excepted.append(mark_excepted(group, reason)) else: - recheck_from_matched.append( - retag_group(group, "voucher_recheck", "PROJECTION_RECONCILE_EXCEPTED_RULE_RELAXED_RECHECK") + next_status = normalized_wehago_status("voucher_recheck", group) + relaxed_from_excepted[next_status].append( + retag_group(group, next_status, "PROJECTION_RECONCILE_EXCEPTED_RULE_RELAXED") ) retained_by_status: dict[str, list[dict[str, Any]]] = {} for status_key in ("voucher_matched", "voucher_unmatched", "voucher_recheck"): @@ -954,17 +1297,16 @@ def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None excepted.append(mark_excepted(group, "MANUAL_OFFSET_PAIR_EXCEPTED")) continue is_excepted, reason = _is_wehago_excepted_voucher_group(group) - if is_excepted and status_key == "voucher_matched" and reason != "WEHAGO_EXCEPTED_DEPRECIATION_ACCOUNT": - recheck_from_matched.append( - retag_group(group, "voucher_recheck", f"PROJECTION_RECONCILE_EXCEPTED_CANDIDATE_RECHECK / {reason}") - ) - elif is_excepted: + if is_excepted: excepted.append(mark_excepted(group, reason)) else: retained.append(group) retained_by_status[status_key] = retained - retained_by_status["voucher_unmatched"].extend(unmatched_from_excepted) - retained_by_status["voucher_recheck"].extend(recheck_from_matched) + for relaxed_status, relaxed_groups in relaxed_from_excepted.items(): + if relaxed_status == "voucher_excepted": + excepted.extend(mark_excepted(group, "") for group in relaxed_groups) + else: + retained_by_status[relaxed_status].extend(relaxed_groups) confirmed = _move_wehago_confirmed_reversal_pairs_to_excepted( { "voucher_matched": retained_by_status["voucher_matched"], @@ -978,13 +1320,14 @@ def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None retained_by_status["voucher_recheck"] = list(confirmed.get("voucher_recheck") or []) excepted = list(confirmed.get("voucher_excepted") or []) apply_net_adjustment_component_matches(retained_by_status) - moved = _move_wehago_offset_tax_invoice_groups_to_excepted( + moved = move_offset_tax_invoice_groups_to_excepted_fast( { "voucher_unmatched": retained_by_status["voucher_unmatched"], "voucher_recheck": retained_by_status["voucher_recheck"], "voucher_excepted": excepted, } ) + moved = move_exact_wehago_reversal_pairs_to_excepted(moved) status_groups["voucher_matched"] = retained_by_status["voucher_matched"] status_groups["voucher_unmatched"] = list(moved.get("voucher_unmatched") or []) status_groups["voucher_recheck"] = list(moved.get("voucher_recheck") or []) @@ -993,12 +1336,1044 @@ def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None ] +def offset_vector_key(vector: dict[tuple[str, str, str], float], *, sign: int = 1) -> tuple[tuple[tuple[str, str, str], float], ...]: + return tuple(sorted((row_key, round(sign * amount, 4)) for row_key, amount in vector.items())) + + +def offset_vector_abs_key(vector: dict[tuple[str, str, str], float]) -> tuple[tuple[tuple[str, str, str], float], ...]: + return tuple(sorted((row_key, round(abs(amount), 4)) for row_key, amount in vector.items())) + + +def group_date_key(group: dict[str, Any]) -> str: + summary = group.get("summary") or {} + fiscal_year = int(summary.get("fiscal_year") or YEAR) + date_text = clean(summary.get("ledger_date")) or clean(summary.get("proof_date")) + if re.fullmatch(r"\d{1,2}[-./]\d{1,2}", date_text): + month, day = re.split(r"[-./]", date_text) + return f"{fiscal_year:04d}-{int(month):02d}-{int(day):02d}" + if re.fullmatch(r"\d{4}[-./]\d{1,2}[-./]\d{1,2}", date_text): + year, month, day = re.split(r"[-./]", date_text) + return f"{int(year):04d}-{int(month):02d}-{int(day):02d}" + return date_text + + +def group_vendor_key(group: dict[str, Any]) -> str: + vendors = { + normalized_compact(row.get("ledger_vendor")) + for row in group.get("rows") or [] + if clean(row.get("ledger_vendor")) + } + if not vendors: + vendors = {normalized_compact((group.get("summary") or {}).get("ledger_vendors"))} + vendors.discard("") + return "|".join(sorted(vendors)) + + +def group_desc_key(group: dict[str, Any]) -> str: + descs = { + normalized_compact(row.get("ledger_desc")) + for row in group.get("rows") or [] + if clean(row.get("ledger_desc")) + } + descs.discard("") + return "|".join(sorted(descs)) + + +def group_reversal_context_key(group: dict[str, Any]) -> tuple[str, str, str]: + return group_date_key(group), group_vendor_key(group), group_desc_key(group) + + +def group_sequence_key(group: dict[str, Any]) -> tuple[str, int]: + identity = group_identity(group) + if re.fullmatch(r"\d{8}-\d{5}", identity): + return identity[:8], int(identity.split("-", 1)[1]) + summary = group.get("summary") or {} + return group_date_key(group).replace("-", ""), int(re.sub(r"\D+", "", clean(summary.get("voucher_no"))) or 0) + + +def group_ledger_direction(group: dict[str, Any]) -> int: + total = 0.0 + for row in group.get("rows") or []: + if not clean(row.get("ledger_account_name")): + continue + total += parse_amount(row.get("ledger_debit")) + total += parse_amount(row.get("ledger_credit")) + if total > 0.5: + return 1 + if total < -0.5: + return -1 + return 0 + + +def group_has_erp_rows(group: dict[str, Any]) -> bool: + return any(has_wehago_value(row) and has_erp_value(row) for row in group.get("rows") or []) + + +def copy_erp_side(source: dict[str, Any], target: dict[str, Any], reason: str) -> dict[str, Any]: + payload = dict(target) + for field in ( + "proof_date", + "draft_no", + "voucher_account_name", + "voucher_vendor", + "voucher_debit", + "voucher_credit", + "voucher_desc", + "voucher_row_key", + ): + payload[field] = source.get(field) + payload["status_label"] = "Matched" + payload["review_reason"] = reason + payload["match_identity_key"] = "|".join( + clean(part) + for part in ( + payload.get("fiscal_year"), + payload.get("voucher_no"), + payload.get("ledger_row_key"), + payload.get("draft_no"), + payload.get("voucher_row_key"), + ) + if clean(part) + ) + return payload + + +def retarget_erp_rows_to_final_reissue(final_group: dict[str, Any], donor_groups: list[dict[str, Any]]) -> dict[str, Any]: + if group_has_erp_rows(final_group): + return final_group + donor_rows = [ + row + for donor in donor_groups + for row in donor.get("rows") or [] + if has_erp_value(row) + ] + if not donor_rows: + return final_group + + used: set[int] = set() + updated_rows: list[dict[str, Any]] = [] + changed = False + for row in final_group.get("rows") or []: + if not has_wehago_value(row) or has_erp_value(row): + updated_rows.append(dict(row)) + continue + ledger_amount = row_side_amount(row, "ledger") + ledger_side = effective_account_side(row, "ledger") + best_index = -1 + best_score: tuple[int, int, float] | None = None + for index, donor_row in enumerate(donor_rows): + if index in used: + continue + if abs(row_side_amount(donor_row, "voucher") - ledger_amount) >= 0.5: + continue + if effective_account_side(donor_row, "voucher") != ledger_side: + continue + if not _account_category_pair_allowed("", row.get("ledger_account_name"), "", donor_row.get("voucher_account_name")): + continue + if not _nature_compatible( + "", + row.get("ledger_account_name"), + ledger_side, + "", + donor_row.get("voucher_account_name"), + ledger_side, + ): + continue + vendor_match = int( + not clean(row.get("ledger_vendor")) + or not clean(donor_row.get("voucher_vendor")) + or normalized_compact(row.get("ledger_vendor")) in normalized_compact(donor_row.get("voucher_vendor")) + or normalized_compact(donor_row.get("voucher_vendor")) in normalized_compact(row.get("ledger_vendor")) + ) + desc_match = int( + not clean(row.get("ledger_desc")) + or not clean(donor_row.get("voucher_desc")) + or normalized_compact(row.get("ledger_desc")) in normalized_compact(donor_row.get("voucher_desc")) + or normalized_compact(donor_row.get("voucher_desc")) in normalized_compact(row.get("ledger_desc")) + ) + score = (vendor_match, desc_match, ledger_amount) + if best_score is None or score > best_score: + best_index = index + best_score = score + if best_index < 0: + updated_rows.append(dict(row)) + continue + used.add(best_index) + updated_rows.append(copy_erp_side(donor_rows[best_index], row, "PROJECTION_RECONCILE_CANCEL_REISSUE_RETARGET_FINAL")) + changed = True + + if not changed: + return final_group + payload = { + "summary": dict(final_group.get("summary") or {}), + "rows": updated_rows, + "source_group_index": final_group.get("source_group_index"), + } + payload["summary"] = rebuild_summary(payload, "voucher_matched") + return payload + + +def apply_cancel_reissue_final_match(status_groups: dict[str, list[dict[str, Any]]]) -> None: + candidates: list[tuple[str, dict[str, Any], tuple[str, str, tuple[tuple[tuple[str, str, str], float], ...]]]] = [] + for status_key in WEHAGO_STATUSES: + if status_key == "voucher_excepted": + continue + for group in status_groups.get(status_key) or []: + vector = _offset_group_vector(group) + if not vector: + continue + vendor_key = group_vendor_key(group) + desc_key = group_desc_key(group) + if not vendor_key or not desc_key: + continue + direction = group_ledger_direction(group) + if direction == 0: + continue + candidates.append((status_key, group, (vendor_key, desc_key, offset_vector_abs_key(vector)))) + if not candidates: + return + + by_chain: dict[tuple[str, str, tuple[tuple[tuple[str, str, str], float], ...]], list[tuple[str, dict[str, Any]]]] = defaultdict(list) + for status_key, group, chain_key in candidates: + by_chain[chain_key].append((status_key, group)) + + move_reasons: dict[int, str] = {} + retargeted: dict[int, dict[str, Any]] = {} + for chain_groups in by_chain.values(): + positives = [(status_key, group) for status_key, group in chain_groups if group_ledger_direction(group) > 0] + negatives = [(status_key, group) for status_key, group in chain_groups if group_ledger_direction(group) < 0] + if not positives or not negatives: + continue + positives_sorted = sorted(positives, key=lambda item: group_sequence_key(item[1])) + final_status, final_group = positives_sorted[-1] + if final_status == "voucher_excepted": + continue + donor_groups = [group for _status_key, group in positives_sorted[:-1] if group_has_erp_rows(group)] + adjusted_final = retarget_erp_rows_to_final_reissue(final_group, donor_groups) + if adjusted_final is not final_group: + retargeted[id(final_group)] = adjusted_final + final_has_erp = group_has_erp_rows(adjusted_final) + if not final_has_erp and not any(group_has_erp_rows(group) for _status_key, group in positives_sorted): + continue + for _status_key, group in negatives: + move_reasons[id(group)] = "WEHAGO_EXCEPTED_CANCEL_REISSUE_CANCEL" + for _status_key, group in positives_sorted[:-1]: + move_reasons[id(group)] = "WEHAGO_EXCEPTED_CANCEL_REISSUE_SUPERSEDED" + + if not move_reasons and not retargeted: + return + + rewritten: dict[str, list[dict[str, Any]]] = defaultdict(list) + for status_key in WEHAGO_STATUSES: + for group in status_groups.get(status_key) or []: + if id(group) in move_reasons: + rewritten["voucher_excepted"].append(mark_excepted(group, move_reasons[id(group)])) + continue + adjusted = retargeted.get(id(group), group) + final_status = normalized_wehago_status(status_key, adjusted) + rewritten[final_status].append(retag_group(adjusted, final_status)) + for status_key in WEHAGO_STATUSES: + status_groups[status_key] = rewritten.get(status_key, []) + + +def accrual_pair_score(ledger_row: dict[str, Any], voucher_row: dict[str, Any]) -> tuple[int, int, int, float]: + if not row_principle_pair_allowed(ledger_row, voucher_row): + return (-1, -1, -1, 0.0) + if abs(row_side_amount(ledger_row, "ledger") - row_side_amount(voucher_row, "voucher")) >= 0.5: + return (-1, -1, -1, 0.0) + same_account = int( + compact_account_name(ledger_row.get("ledger_account_name")) == compact_account_name(voucher_row.get("voucher_account_name")) + or compact_account_name(ledger_row.get("ledger_account_name")) in compact_account_name(voucher_row.get("voucher_account_name")) + or compact_account_name(voucher_row.get("voucher_account_name")) in compact_account_name(ledger_row.get("ledger_account_name")) + ) + vendor_match = int( + not clean(ledger_row.get("ledger_vendor")) + or not clean(voucher_row.get("voucher_vendor")) + or normalized_compact(ledger_row.get("ledger_vendor")) in normalized_compact(voucher_row.get("voucher_vendor")) + or normalized_compact(voucher_row.get("voucher_vendor")) in normalized_compact(ledger_row.get("ledger_vendor")) + ) + desc_match = int( + not clean(ledger_row.get("ledger_desc")) + or not clean(voucher_row.get("voucher_desc")) + or normalized_compact(ledger_row.get("ledger_desc")) in normalized_compact(voucher_row.get("voucher_desc")) + or normalized_compact(voucher_row.get("voucher_desc")) in normalized_compact(ledger_row.get("ledger_desc")) + ) + return same_account, vendor_match, desc_match, row_side_amount(ledger_row, "ledger") + + +def merge_accrual_pair(ledger_row: dict[str, Any], voucher_row: dict[str, Any], reason: str) -> dict[str, Any]: + payload = dict(ledger_row) + for field in ( + "proof_date", + "draft_no", + "voucher_account_name", + "voucher_vendor", + "voucher_debit", + "voucher_credit", + "voucher_desc", + "voucher_row_key", + ): + payload[field] = voucher_row.get(field) + payload["status_label"] = "Matched" + payload["review_reason"] = reason + payload["match_identity_key"] = "|".join( + clean(part) + for part in ( + payload.get("fiscal_year"), + payload.get("voucher_no"), + payload.get("ledger_row_key"), + payload.get("draft_no"), + payload.get("voucher_row_key"), + ) + if clean(part) + ) + return payload + + +def remove_recheck_reason_tokens(row: dict[str, Any], reason: str) -> dict[str, Any]: + blocked = { + "PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP", + "PROJECTION_RECONCILE_PARTIAL_MATCH_WEHAGO_ROW_UNMATCHED", + "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE", + } + kept: list[str] = [] + append_review_reason_parts(kept, row.get("review_reason")) + kept = [part for part in kept if part not in blocked] + append_review_reason_parts(kept, reason) + payload = dict(row) + payload["review_reason"] = " / ".join(kept) + payload["status_label"] = "Matched" + return payload + + +def strip_review_reason_tokens_from_group(group: dict[str, Any], blocked: set[str]) -> dict[str, Any]: + payload = { + "summary": dict(group.get("summary") or {}), + "rows": [dict(row) for row in group.get("rows") or []], + "source_group_index": group.get("source_group_index"), + } + for target in [payload["summary"], *payload["rows"]]: + parts: list[str] = [] + append_review_reason_parts(parts, target.get("review_reason")) + target["review_reason"] = " / ".join(part for part in parts if part not in blocked) + return payload + + +def build_accrual_principle_match(group: dict[str, Any], reason: str) -> dict[str, Any] | None: + rows = [dict(row) for row in group.get("rows") or []] + ledger_business = [ + row for row in rows + if has_wehago_value(row) and (is_business_row(row, "ledger") or is_vat_row(row, "ledger")) + ] + voucher_business = [ + row for row in rows + if has_erp_value(row) and (is_business_row(row, "voucher") or is_vat_row(row, "voucher")) + ] + if not ledger_business or not voucher_business: + return None + + pair_candidates: list[tuple[tuple[int, int, int, float], int, int]] = [] + for ledger_index, ledger_row in enumerate(ledger_business): + for voucher_index, voucher_row in enumerate(voucher_business): + score = accrual_pair_score(ledger_row, voucher_row) + if score[0] < 0: + continue + pair_candidates.append((score, ledger_index, voucher_index)) + if not pair_candidates: + return None + pair_candidates.sort(reverse=True) + + used_ledger: set[int] = set() + used_voucher: set[int] = set() + merged_rows: list[dict[str, Any]] = [] + covered_ledger_keys: set[tuple[str, str, str, str]] = set() + covered_voucher_keys: set[tuple[str, str, str, str]] = set() + for _score, ledger_index, voucher_index in pair_candidates: + if ledger_index in used_ledger or voucher_index in used_voucher: + continue + ledger_row = ledger_business[ledger_index] + voucher_row = voucher_business[voucher_index] + used_ledger.add(ledger_index) + used_voucher.add(voucher_index) + covered_ledger_keys.add(row_business_match_key(ledger_row, "ledger")) + covered_voucher_keys.add(row_business_match_key(voucher_row, "voucher")) + merged_rows.append(merge_accrual_pair(ledger_row, voucher_row, reason)) + + if not merged_rows: + return None + + unmatched_voucher_business = [ + row for index, row in enumerate(voucher_business) + if index not in used_voucher and row_business_match_key(row, "voucher") not in covered_voucher_keys + ] + if unmatched_voucher_business: + return None + + for index, ledger_row in enumerate(ledger_business): + if index in used_ledger: + continue + ledger_key = row_business_match_key(ledger_row, "ledger") + if ledger_key in covered_ledger_keys: + continue + if row_principle_signature(ledger_row, "ledger").endswith(":decrease"): + continue + return None + + context_rows: list[dict[str, Any]] = [] + seen_context: set[tuple[Any, ...]] = set() + for row in rows: + if has_wehago_value(row) and (is_business_row(row, "ledger") or is_vat_row(row, "ledger")): + key = row_business_match_key(row, "ledger") + if key in covered_ledger_keys or row_principle_signature(row, "ledger").endswith(":decrease"): + continue + return None + if has_erp_value(row) and (is_business_row(row, "voucher") or is_vat_row(row, "voucher")): + key = row_business_match_key(row, "voucher") + if key in covered_voucher_keys: + continue + return None + ledger_context = has_wehago_value(row) and is_settlement_row(row, "ledger") + voucher_context = has_erp_value(row) and is_settlement_row(row, "voucher") + if not ledger_context and not voucher_context: + continue + context = remove_recheck_reason_tokens(row, reason) + context_key = ( + ledger_row_identity(context) if has_wehago_value(context) else None, + voucher_row_identity(context) if has_erp_value(context) else None, + ) + if context_key in seen_context: + continue + seen_context.add(context_key) + context_rows.append(context) + + payload = { + "summary": dict(group.get("summary") or {}), + "rows": merged_rows + context_rows, + "source_group_index": group.get("source_group_index"), + } + payload["summary"] = rebuild_summary(payload, "voucher_matched") + return payload + + +def index_erp_matched_accrual_donors(groups: dict[str, list[dict[str, Any]]]) -> dict[str, list[dict[str, Any]]]: + donors: dict[str, list[dict[str, Any]]] = defaultdict(list) + for group in groups.get("erp_voucher_matched") or []: + identity = group_identity(group) + if not identity: + continue + donors[identity].append(group) + return donors + + +def build_requested_draft_donor_match(group: dict[str, Any], donor: dict[str, Any]) -> dict[str, Any] | None: + requested_bases = row_draft_bases(group.get("rows") or [], group.get("summary") or {}) + if not requested_bases: + return None + filtered_rows = [ + dict(row) + for row in donor.get("rows") or [] + if clean(row.get("draft_no")) and erp_voucher_base(row.get("draft_no")) in requested_bases + ] + if not filtered_rows: + return None + filtered = { + "summary": dict(donor.get("summary") or {}), + "rows": filtered_rows, + "source_group_index": donor.get("source_group_index"), + } + filtered["summary"] = rebuild_summary(filtered, "voucher_matched") + return build_accrual_principle_match(filtered, "PROJECTION_RECONCILE_ACCRUAL_PRINCIPLE_ERP_DONOR_MATCH") + + +def apply_accrual_principle_matches( + status_groups: dict[str, list[dict[str, Any]]], + source_groups: dict[str, list[dict[str, Any]]], +) -> None: + donors_by_identity = index_erp_matched_accrual_donors(source_groups) + rewritten: dict[str, list[dict[str, Any]]] = defaultdict(list) + for status_key in WEHAGO_STATUSES: + for group in status_groups.get(status_key) or []: + if status_key != "voucher_recheck": + rewritten[status_key].append(group) + continue + promoted = build_accrual_principle_match(group, "PROJECTION_RECONCILE_ACCRUAL_PRINCIPLE_BUSINESS_MATCH") + if promoted is None: + donors = [ + donor_match + for donor in donors_by_identity.get(group_identity(group), []) + for donor_match in [build_requested_draft_donor_match(group, donor)] + if donor_match is not None + ] + if donors: + promoted = max( + donors, + key=lambda donor: ( + sum(1 for row in donor.get("rows") or [] if has_wehago_value(row) and has_erp_value(row)), + parse_amount((donor.get("summary") or {}).get("ledger_debit")) + parse_amount((donor.get("summary") or {}).get("ledger_credit")), + ), + ) + if promoted is None: + rewritten[status_key].append( + strip_review_reason_tokens_from_group( + group, + { + "PROJECTION_RECONCILE_ACCRUAL_PRINCIPLE_BUSINESS_MATCH", + "PROJECTION_RECONCILE_ACCRUAL_PRINCIPLE_ERP_DONOR_MATCH", + }, + ) + ) + continue + rewritten["voucher_matched"].append(retag_group(promoted, "voucher_matched")) + for status_key in WEHAGO_STATUSES: + status_groups[status_key] = rewritten.get(status_key, []) + + +def erp_entry_draft_year(entry: dict[str, Any]) -> int: + for value in (entry.get("draft_no"), entry.get("confirmed_no")): + match = re.search(r"11-(\d{4})\d{4}-", clean(value)) + if match: + return int(match.group(1)) + return int(entry.get("fiscal_year") or 0) + + +def raw_erp_entry_identity(entry: dict[str, Any]) -> tuple[Any, ...]: + return ( + entry.get("id"), + clean(entry.get("draft_no")), + clean(entry.get("confirmed_no")), + int(entry.get("row_number") or 0), + clean(entry.get("account_code")), + ) + + +def raw_erp_entry_draft_identity(entry: dict[str, Any]) -> tuple[str, str]: + return ("draft", clean(entry.get("draft_no")) or clean(entry.get("confirmed_no"))) + + +def raw_erp_entry_text(entry: dict[str, Any]) -> str: + return " ".join( + clean(part) + for part in ( + entry.get("account_name"), + entry.get("vendor_name"), + entry.get("desc1"), + entry.get("desc2"), + entry.get("management_item"), + ) + if clean(part) + ) + + +def compact_text_tokens(value: Any) -> set[str]: + tokens = { + token + for token in re.split(r"[^0-9a-z가-힣]+", normalized_compact(value)) + if len(token) >= 2 + } + return tokens + + +def compact_meaningful_parts(value: Any) -> list[str]: + parts: list[str] = [] + for token in re.split(r"[^0-9a-zA-Z가-힣]+", clean(value)): + compact = normalized_compact(token) + if len(compact) >= 4: + parts.append(compact) + return parts + + +def row_desc_matches_raw_entry(row: dict[str, Any], entry: dict[str, Any]) -> bool: + ledger_parts = compact_meaningful_parts(row.get("ledger_desc")) + entry_text = normalized_compact(" ".join([clean(entry.get("desc1")), clean(entry.get("desc2")), clean(entry.get("management_item"))])) + if not ledger_parts or not entry_text: + return False + return any(part in entry_text or entry_text in part for part in ledger_parts) + + +def row_text_matches_raw_entry(row: dict[str, Any], entry: dict[str, Any]) -> bool: + if row_desc_matches_raw_entry(row, entry): + return True + ledger_vendor = normalized_compact(row.get("ledger_vendor")) + entry_vendor = normalized_compact(entry.get("vendor_name")) + if ledger_vendor and entry_vendor and (ledger_vendor in entry_vendor or entry_vendor in ledger_vendor): + return True + ledger_text = " ".join( + clean(part) + for part in (row.get("ledger_account_name"), row.get("ledger_vendor"), row.get("ledger_desc")) + if clean(part) + ) + return bool(compact_text_tokens(ledger_text) & compact_text_tokens(raw_erp_entry_text(entry))) + + +def raw_entry_candidate_row(entry: dict[str, Any], ledger_row: dict[str, Any], reason: str) -> dict[str, Any] | None: + category = _classify_account_category(entry.get("account_code"), entry.get("account_name")) + signed_amount, side = raw_erp_entry_signed_amount_side(entry, category) + if abs(signed_amount) <= 0: + return None + ledger_debit = parse_amount(ledger_row.get("ledger_debit")) + ledger_credit = parse_amount(ledger_row.get("ledger_credit")) + if side == "debit" and abs(ledger_debit - signed_amount) >= 0.5: + return None + if side == "credit" and abs(ledger_credit - signed_amount) >= 0.5: + return None + payload = raw_erp_entry_to_row(entry, ledger_row, side, reason) + payload["voucher_debit"] = signed_amount if side == "debit" else 0 + payload["voucher_credit"] = signed_amount if side == "credit" else 0 + payload["voucher_row_key"] = build_voucher_row_key(payload) + if not row_principle_pair_allowed(ledger_row, payload): + return None + return payload + + +def split_draft_raw_candidate_score( + ledger_row: dict[str, Any], + entry: dict[str, Any], + fiscal_year: int, +) -> tuple[int, int, int, int, int, float]: + proof_match = int(dates_match_wehago(entry.get("proof_date"), ledger_row.get("ledger_date"), fiscal_year)) + desc_match = int(row_desc_matches_raw_entry(ledger_row, entry)) + same_account = int( + compact_account_name(ledger_row.get("ledger_account_name")) == compact_account_name(entry.get("account_name")) + or compact_account_name(ledger_row.get("ledger_account_name")) in compact_account_name(entry.get("account_name")) + or compact_account_name(entry.get("account_name")) in compact_account_name(ledger_row.get("ledger_account_name")) + ) + text_match = int(row_text_matches_raw_entry(ledger_row, entry)) + current_year = int(erp_entry_draft_year(entry) == fiscal_year) + return proof_match, desc_match, same_account, text_match, current_year, row_side_amount(ledger_row, "ledger") + + +def find_split_draft_raw_match( + ledger_row: dict[str, Any], + draft_bases: set[str], + used_entries: set[tuple[Any, ...]], + fiscal_year: int, +) -> dict[str, Any] | None: + best: tuple[tuple[int, int, int, int, float], dict[str, Any]] | None = None + for base in sorted(draft_bases): + for entry in RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, []): + entry_identity = raw_erp_entry_identity(entry) + if entry_identity in used_entries or raw_erp_entry_draft_identity(entry) in used_entries: + continue + entry_year = erp_entry_draft_year(entry) + if entry_year and entry_year != fiscal_year: + continue + candidate = raw_entry_candidate_row( + entry, + ledger_row, + "PROJECTION_RECONCILE_SPLIT_DRAFT_ROW_MATCH", + ) + if candidate is None: + continue + score = split_draft_raw_candidate_score(ledger_row, entry, fiscal_year) + if not any(score[:4]): + continue + if best is None or score > best[0]: + best = (score, candidate) + if best is None: + return None + return best[1] + + +def row_existing_erp_is_material_match(row: dict[str, Any]) -> bool: + if not (has_wehago_value(row) and has_erp_value(row)): + return False + if abs(row_side_amount(row, "ledger") - row_side_amount(row, "voucher")) >= 0.5: + return False + return row_principle_pair_allowed(row, row) + + +def build_split_draft_row_match(group: dict[str, Any]) -> dict[str, Any] | None: + rows = [dict(row) for row in group.get("rows") or []] + summary = dict(group.get("summary") or {}) + draft_bases = row_draft_bases(rows, summary) + if not draft_bases: + return None + fiscal_year = int(summary.get("fiscal_year") or YEAR) + material_rows = [ + row + for row in rows + if has_wehago_value(row) and (is_business_row(row, "ledger") or is_vat_row(row, "ledger")) + ] + if not material_rows: + return None + + matched_rows: list[dict[str, Any]] = [] + used_entries: set[tuple[Any, ...]] = set() + covered_material_indexes: set[int] = set() + covered_material_keys: set[tuple[str, str, str, str]] = set() + raw_needed: list[tuple[int, dict[str, Any]]] = [] + for material_index, row in enumerate(material_rows): + if row_existing_erp_is_material_match(row): + matched = remove_recheck_reason_tokens(row, "PROJECTION_RECONCILE_SPLIT_DRAFT_EXISTING_ROW_MATCH") + matched_rows.append(matched) + covered_material_indexes.add(material_index) + covered_material_keys.add(row_business_match_key(row, "ledger")) + if clean(row.get("draft_no")): + used_entries.add(("draft", clean(row.get("draft_no")))) + continue + raw_needed.append((material_index, row)) + + raw_pair_candidates: list[tuple[tuple[int, int, int, int, int, float], int, dict[str, Any], tuple[Any, ...]]] = [] + for material_index, row in raw_needed: + for base in sorted(draft_bases): + for entry in RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, []): + entry_identity = raw_erp_entry_identity(entry) + draft_identity = raw_erp_entry_draft_identity(entry) + if entry_identity in used_entries or draft_identity in used_entries: + continue + entry_year = erp_entry_draft_year(entry) + if entry_year and entry_year != fiscal_year: + continue + candidate = raw_entry_candidate_row( + entry, + row, + "PROJECTION_RECONCILE_SPLIT_DRAFT_ROW_MATCH", + ) + if candidate is None: + continue + score = split_draft_raw_candidate_score(row, entry, fiscal_year) + if not any(score[:4]): + continue + raw_pair_candidates.append((score, material_index, candidate, draft_identity)) + raw_pair_candidates.sort(key=lambda item: (item[0], -item[1]), reverse=True) + + for _score, material_index, candidate, draft_identity in raw_pair_candidates: + if material_index in covered_material_indexes or draft_identity in used_entries: + continue + matched_rows.append(candidate) + covered_material_indexes.add(material_index) + covered_material_keys.add(row_business_match_key(material_rows[material_index], "ledger")) + used_entries.add(draft_identity) + + for material_index, row in enumerate(material_rows): + if material_index in covered_material_indexes: + continue + if row_business_match_key(row, "ledger") in covered_material_keys: + continue + return None + + context_rows: list[dict[str, Any]] = [] + seen_context: set[tuple[Any, ...]] = set() + for row in rows: + if has_wehago_value(row) and (is_business_row(row, "ledger") or is_vat_row(row, "ledger")): + continue + if has_erp_value(row) and not has_wehago_value(row): + continue + if not has_wehago_value(row): + continue + context = remove_recheck_reason_tokens(row, "PROJECTION_RECONCILE_SPLIT_DRAFT_CONTEXT_ROW") + context_key = ledger_row_identity(context) + if context_key in seen_context: + continue + seen_context.add(context_key) + context_rows.append(context) + + payload = { + "summary": summary, + "rows": matched_rows + context_rows, + "source_group_index": group.get("source_group_index"), + } + payload["summary"] = rebuild_summary(payload, "voucher_matched") + return payload + + +def apply_split_draft_row_matches(status_groups: dict[str, list[dict[str, Any]]]) -> None: + rewritten: dict[str, list[dict[str, Any]]] = defaultdict(list) + for status_key in WEHAGO_STATUSES: + for group in status_groups.get(status_key) or []: + if status_key != "voucher_recheck": + rewritten[status_key].append(group) + continue + promoted = build_split_draft_row_match(group) + if promoted is None: + rewritten[status_key].append( + strip_review_reason_tokens_from_group( + group, + { + "PROJECTION_RECONCILE_SPLIT_DRAFT_EXISTING_ROW_MATCH", + "PROJECTION_RECONCILE_SPLIT_DRAFT_ROW_MATCH", + "PROJECTION_RECONCILE_SPLIT_DRAFT_CONTEXT_ROW", + }, + ) + ) + continue + rewritten["voucher_matched"].append(retag_group(promoted, "voucher_matched")) + for status_key in WEHAGO_STATUSES: + status_groups[status_key] = rewritten.get(status_key, []) + + +def extract_dates_from_text(value: Any) -> set[str]: + text = clean(value) + dates: set[str] = set() + for year, month, day in re.findall(r"((?:19|20)\d{2})[-./년\s]*(\d{1,2})[-./월\s]*(\d{1,2})", text): + dates.add(f"{int(year):04d}-{int(month):02d}-{int(day):02d}") + return dates + + +def raw_erp_unique_entries() -> list[dict[str, Any]]: + seen: set[Any] = set() + entries: list[dict[str, Any]] = [] + for rows in RAW_ERP_ROWS_BY_DRAFT_BASE.values(): + for row in rows: + key = row.get("id") or ( + row.get("draft_no"), + row.get("confirmed_no"), + row.get("account_code"), + row.get("row_number"), + ) + if key in seen: + continue + seen.add(key) + entries.append(row) + return entries + + +def group_has_management_item_trace(group: dict[str, Any]) -> bool: + date_key, vendor_key, desc_key = group_reversal_context_key(group) + if not date_key or not vendor_key: + return False + group_amounts = { + amount_key(abs(parse_amount(row.get("ledger_debit")) or parse_amount(row.get("ledger_credit")))) + for row in group.get("rows") or [] + if abs(parse_amount(row.get("ledger_debit")) or parse_amount(row.get("ledger_credit"))) >= 0.5 + } + for entry in raw_erp_unique_entries(): + management_text = clean(entry.get("management_item")) + if not management_text: + continue + entry_vendor = normalized_compact(entry.get("vendor_name")) + if entry_vendor and vendor_key and entry_vendor not in vendor_key and vendor_key not in entry_vendor: + continue + entry_dates = extract_dates_from_text(management_text) + proof_date = clean(entry.get("proof_date")) + if proof_date: + entry_dates.add(proof_date) + confirmed = clean(entry.get("confirmed_no")) + matched = re.search(r"11-((?:19|20)\d{6})-", confirmed) + if matched: + raw = matched.group(1) + entry_dates.add(f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}") + if date_key not in entry_dates: + continue + entry_amounts = { + amount_key(abs(parse_amount(entry.get(field)))) + for field in ("debit_supply", "debit_tax", "credit_supply", "credit_tax") + if abs(parse_amount(entry.get(field))) >= 0.5 + } + if group_amounts and entry_amounts and group_amounts.isdisjoint(entry_amounts): + if desc_key and normalized_compact(entry.get("desc1")) not in desc_key and desc_key not in normalized_compact(entry.get("desc1")): + continue + return True + return False + + +def move_exact_wehago_reversal_pairs_to_excepted( + voucher_sections: dict[str, list[dict[str, Any]]], +) -> dict[str, list[dict[str, Any]]]: + candidate_statuses = ("voucher_unmatched", "voucher_recheck") + candidate_groups: list[tuple[str, dict[str, Any]]] = [ + (status_key, group) + for status_key in candidate_statuses + for group in list(voucher_sections.get(status_key) or []) + ] + if len(candidate_groups) < 2: + return voucher_sections + + vectors: dict[int, dict[tuple[str, str, str], float]] = {} + indexed: dict[tuple[tuple[str, str, str], tuple[tuple[tuple[str, str, str], float], ...]], list[tuple[str, dict[str, Any]]]] = defaultdict(list) + for status_key, group in candidate_groups: + vector = _offset_group_vector(group) + context_key = group_reversal_context_key(group) + if not vector or not all(context_key): + continue + vectors[id(group)] = vector + indexed[(context_key, offset_vector_key(vector))].append((status_key, group)) + + moved_ids: set[int] = set() + move_reasons: dict[int, str] = {} + for status_key, group in candidate_groups: + if id(group) in moved_ids: + continue + vector = vectors.get(id(group)) + if not vector: + continue + context_key = group_reversal_context_key(group) + for _other_status, other in indexed.get((context_key, offset_vector_key(vector, sign=-1)), []): + if other is group or id(other) in moved_ids: + continue + reason = ( + "WEHAGO_EXCEPTED_MANAGEMENT_ITEM_REVERSAL_PAIR" + if group_has_management_item_trace(group) or group_has_management_item_trace(other) + else "WEHAGO_EXCEPTED_EXACT_REVERSAL_PAIR" + ) + moved_ids.update((id(group), id(other))) + move_reasons[id(group)] = reason + move_reasons[id(other)] = reason + break + + if not moved_ids: + return voucher_sections + + retained_by_status: dict[str, list[dict[str, Any]]] = {status_key: [] for status_key in candidate_statuses} + moved_excepted: list[dict[str, Any]] = [] + for status_key, group in candidate_groups: + if id(group) not in moved_ids: + retained_by_status[status_key].append(group) + continue + moved_excepted.append(mark_excepted(group, move_reasons.get(id(group), "WEHAGO_EXCEPTED_EXACT_REVERSAL_PAIR"))) + for status_key in candidate_statuses: + voucher_sections[status_key] = retained_by_status[status_key] + voucher_sections["voucher_excepted"] = list(voucher_sections.get("voucher_excepted") or []) + moved_excepted + return voucher_sections + + +def move_offset_tax_invoice_groups_to_excepted_fast( + voucher_sections: dict[str, list[dict[str, Any]]], +) -> dict[str, list[dict[str, Any]]]: + candidate_statuses = ("voucher_unmatched", "voucher_recheck") + candidate_groups: list[tuple[str, dict[str, Any]]] = [ + (status_key, group) + for status_key in candidate_statuses + for group in list(voucher_sections.get(status_key) or []) + ] + if len(candidate_groups) < 2: + return voucher_sections + + candidate_vectors: dict[int, dict[tuple[str, str, str], float]] = {} + tax_offset_candidate_ids: set[int] = set() + indexed: dict[tuple[tuple[tuple[str, str, str], float], ...], list[tuple[str, dict[str, Any]]]] = defaultdict(list) + for status_key, group in candidate_groups: + if _voucher_group_has_review_reason( + group, + "MATCHED_CANCEL_TARGET_RECHECK", + "CANCEL_TARGET_ALREADY_MATCHED_RECHECK", + "CANCEL_REISSUE_RETARGET_RECHECK", + ): + continue + if _group_has_tax_invoice_cancel_signal(group) or _group_has_offset_tax_invoice_structure(group): + tax_offset_candidate_ids.add(id(group)) + vector = _offset_group_vector(group) + if not vector: + continue + candidate_vectors[id(group)] = vector + indexed[offset_vector_key(vector)].append((status_key, group)) + if not candidate_vectors: + return voucher_sections + + moved_ids: set[int] = set() + for status_key, group in candidate_groups: + if id(group) in moved_ids: + continue + left_vector = candidate_vectors.get(id(group)) + if not left_vector: + continue + left_dates = _voucher_group_month_days(group) + for _other_status_key, other in indexed.get(offset_vector_key(left_vector, sign=-1), []): + if other is group or id(other) in moved_ids: + continue + same_date = bool(left_dates & _voucher_group_month_days(other)) + is_tax_offset_pair = id(group) in tax_offset_candidate_ids and id(other) in tax_offset_candidate_ids + is_near_reversal_pair = _voucher_groups_within_days(group, other, 62) + if not (same_date or is_tax_offset_pair or is_near_reversal_pair): + continue + moved_ids.add(id(group)) + moved_ids.add(id(other)) + break + + if not moved_ids: + return voucher_sections + + retained_by_status: dict[str, list[dict[str, Any]]] = {status_key: [] for status_key in candidate_statuses} + moved_excepted: list[dict[str, Any]] = [] + for status_key, group in candidate_groups: + if id(group) not in moved_ids: + retained_by_status[status_key].append(group) + continue + moved_excepted.append(mark_excepted(group, "WEHAGO_EXCEPTED_OFFSET_REVERSAL_PAIR")) + + for status_key in candidate_statuses: + voucher_sections[status_key] = retained_by_status[status_key] + voucher_sections["voucher_excepted"] = list(voucher_sections.get("voucher_excepted") or []) + moved_excepted + return voucher_sections + + +def apply_split_revenue_matches_to_status_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None: + updated: dict[str, list[dict[str, Any]]] = defaultdict(list) + for status_key in WEHAGO_STATUSES: + for group in list(status_groups.get(status_key) or []): + if status_key not in {"voucher_matched", "voucher_recheck"}: + updated[status_key].append(group) + continue + summary = dict(group.get("summary") or {}) + rows = apply_same_draft_split_revenue_matches([dict(row) for row in group.get("rows") or []], summary) + adjusted = {"summary": summary, "rows": rows, "source_group_index": group.get("source_group_index")} + adjusted["summary"] = rebuild_summary(adjusted, status_key) + final_status = normalized_wehago_status(status_key, adjusted) + updated[final_status].append(retag_group(adjusted, final_status)) + for status_key in WEHAGO_STATUSES: + status_groups[status_key] = updated.get(status_key, []) + + +def apply_exact_reversal_pairs_to_status_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None: + moved = move_exact_wehago_reversal_pairs_to_excepted( + { + "voucher_unmatched": list(status_groups.get("voucher_unmatched") or []), + "voucher_recheck": list(status_groups.get("voucher_recheck") or []), + "voucher_excepted": list(status_groups.get("voucher_excepted") or []), + } + ) + status_groups["voucher_unmatched"] = list(moved.get("voucher_unmatched") or []) + status_groups["voucher_recheck"] = list(moved.get("voucher_recheck") or []) + status_groups["voucher_excepted"] = list(moved.get("voucher_excepted") or []) + + +def enforce_wehago_status_invariants(status_groups: dict[str, list[dict[str, Any]]]) -> dict[str, int]: + diagnostics = Counter() + reclassified: dict[str, list[dict[str, Any]]] = defaultdict(list) + for status_key in WEHAGO_STATUSES: + for group in list(status_groups.get(status_key) or []): + final_status = normalized_wehago_status(status_key, group) + if final_status != status_key: + diagnostics[f"{status_key}_to_{final_status}"] += 1 + if final_status == "voucher_excepted": + is_excepted, reason = _is_wehago_excepted_voucher_group(group) + if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES: + reason = reason or "MANUAL_OFFSET_PAIR_EXCEPTED" + reclassified[final_status].append(mark_excepted(group, reason)) + else: + reclassified[final_status].append(retag_group(group, final_status)) + for status_key in WEHAGO_STATUSES: + status_groups[status_key] = reclassified.get(status_key, []) + diagnostics["voucher_recheck_without_erp"] = sum( + 1 for group in status_groups.get("voucher_recheck") or [] if not group_has_real_erp_candidate(group) + ) + diagnostics["voucher_matched_partial_wehago"] = sum( + 1 for group in status_groups.get("voucher_matched") or [] if group_has_unmatched_wehago_rows(group) + ) + diagnostics["voucher_unmatched_with_erp"] = sum( + 1 for group in status_groups.get("voucher_unmatched") or [] if group_has_real_erp_candidate(group) + ) + return {key: int(value) for key, value in diagnostics.items()} + + def append_unique(values: list[str], value: Any) -> None: text = clean(value) if text and text not in values: values.append(text) +def append_review_reason_parts(values: list[str], value: Any) -> None: + for part in re.split(r"\s*/\s*", clean(value)): + append_unique(values, part) + + +def dedup_review_reason_text(*values: Any) -> str: + parts: list[str] = [] + for value in values: + append_review_reason_parts(parts, value) + return " / ".join(parts) + + def rebuild_summary(group: dict[str, Any], status_key: str) -> dict[str, Any]: old = dict(group.get("summary") or {}) rows = list(group.get("rows") or []) @@ -1031,7 +2406,7 @@ def rebuild_summary(group: dict[str, Any], status_key: str) -> dict[str, Any]: append_unique(voucher_accounts, row.get("voucher_account_name")) append_unique(voucher_vendors, row.get("voucher_vendor")) append_unique(draft_nos, row.get("draft_no")) - append_unique(reasons, row.get("review_reason")) + append_review_reason_parts(reasons, row.get("review_reason")) if not clean(summary.get("ledger_date")) and clean(row.get("ledger_date")): summary["ledger_date"] = clean(row.get("ledger_date")) if not clean(summary.get("proof_date")) and clean(row.get("proof_date")): @@ -1042,7 +2417,9 @@ def rebuild_summary(group: dict[str, Any], status_key: str) -> dict[str, Any]: summary["ledger_vendors"] = ", ".join(ledger_vendors) summary["voucher_vendors"] = ", ".join(voucher_vendors) summary["draft_no"] = ", ".join(draft_nos) or clean(old.get("draft_no")) - summary["review_reason"] = " / ".join(reasons) or clean(old.get("review_reason")) + if not reasons: + append_review_reason_parts(reasons, old.get("review_reason")) + summary["review_reason"] = " / ".join(reasons) summary["search_text"] = " ".join( clean(part) for part in [ @@ -1308,6 +2685,188 @@ def update_snapshot_row_counts(conn: sqlite3.Connection, counts: dict[str, int]) ) +def ensure_compare_settings(conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS wehago_compare_settings ( + setting_key TEXT PRIMARY KEY, + setting_json TEXT NOT NULL DEFAULT '{}', + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + + +def activate_projection_signature( + conn: sqlite3.Connection, + signature: str, + counts: dict[str, int], + diagnostics: dict[str, int], +) -> None: + ensure_compare_settings(conn) + payload = { + "start_year": YEAR, + "end_year": YEAR, + "signature": signature, + "logic_version": RECONCILED_PROJECTION_VERSION, + "counts": {key: int(counts.get(key, 0) or 0) for key in ALL_VOUCHER_STATUSES}, + "diagnostics": diagnostics, + "activated_at": datetime.now().isoformat(timespec="seconds"), + } + conn.execute( + """ + INSERT INTO wehago_compare_settings (setting_key, setting_json, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(setting_key) DO UPDATE SET + setting_json = excluded.setting_json, + updated_at = CURRENT_TIMESTAMP + """, + ( + f"wehago_active_query_projection:{YEAR}:{YEAR}", + json.dumps(payload, ensure_ascii=False), + ), + ) + + +def store_query_metric_projection(conn: sqlite3.Connection, signature: str, counts: dict[str, int]) -> None: + conn.execute( + """ + INSERT INTO wehago_compare_query_metrics ( + start_year, end_year, signature, counts_json, snapshot_state_json, source_state_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT(start_year, end_year, signature) DO UPDATE SET + counts_json = excluded.counts_json, + snapshot_state_json = excluded.snapshot_state_json, + source_state_json = excluded.source_state_json, + updated_at = CURRENT_TIMESTAMP + """, + ( + YEAR, + YEAR, + signature, + json.dumps({key: int(counts.get(key, 0) or 0) for key in ALL_VOUCHER_STATUSES}, ensure_ascii=False), + json.dumps({"ready": [YEAR], "missing": [], "stale": [], "queued": [], "running": []}, ensure_ascii=False), + json.dumps({"projection_signature": signature, "source": "reconcile_wehago_projection_to_db"}, ensure_ascii=False), + ), + ) + + +def backfill_final_status_projection(conn: sqlite3.Connection, signature: str) -> int: + priority = { + "voucher_excepted": 0, + "voucher_matched": 1, + "voucher_recheck": 2, + "voucher_unmatched": 3, + } + priority_case = " ".join(f"WHEN '{status}' THEN {rank}" for status, rank in priority.items()) + status_case = " ".join(f"WHEN {rank} THEN '{status}'" for status, rank in priority.items()) + conn.execute( + """ + DELETE FROM wehago_compare_final_status_projection + WHERE start_year = ? AND end_year = ? AND signature = ? + """, + (YEAR, YEAR, signature), + ) + conn.execute( + f""" + INSERT INTO wehago_compare_final_status_projection ( + start_year, end_year, signature, identity_key, compare_voucher_no, + fiscal_year, ledger_date, voucher_no, final_status, final_rank, + source_statuses, source_group_count, created_at, updated_at + ) + WITH source_groups AS ( + SELECT + fiscal_year, + status_key, + CASE status_key {priority_case} ELSE 99 END AS status_rank, + COALESCE(NULLIF(TRIM(ledger_date), ''), NULLIF(TRIM(proof_date), ''), '') AS raw_date, + COALESCE(NULLIF(TRIM(voucher_no), ''), CAST(group_index AS TEXT)) AS raw_voucher_no + FROM wehago_compare_query_groups + WHERE start_year = ? + AND end_year = ? + AND signature = ? + AND status_key IN ('voucher_matched', 'voucher_recheck', 'voucher_unmatched', 'voucher_excepted') + ), + normalized AS ( + SELECT + fiscal_year, + status_key, + status_rank, + CASE + WHEN raw_date GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' THEN raw_date + WHEN raw_date GLOB '[0-9][0-9]-[0-9][0-9]' THEN printf('%04d-%s', fiscal_year, raw_date) + WHEN raw_date GLOB '[0-9]-[0-9][0-9]' THEN printf('%04d-0%s', fiscal_year, raw_date) + ELSE raw_date + END AS ledger_date, + TRIM(CASE WHEN INSTR(raw_voucher_no, ',') > 0 THEN SUBSTR(raw_voucher_no, 1, INSTR(raw_voucher_no, ',') - 1) ELSE raw_voucher_no END) AS voucher_no + FROM source_groups + ), + classified AS ( + SELECT + status_key, + status_rank, + fiscal_year, + ledger_date, + voucher_no, + CAST(fiscal_year AS TEXT) || '|' || ledger_date || '|' || voucher_no AS identity_key, + CASE WHEN ledger_date <> '' AND voucher_no <> '' THEN REPLACE(ledger_date, '-', '') || '-' || voucher_no ELSE '' END AS compare_voucher_no + FROM normalized + WHERE COALESCE(voucher_no, '') <> '' + ), + resolved AS ( + SELECT + identity_key, + MIN(status_rank) AS final_rank, + MIN(fiscal_year) AS fiscal_year, + MIN(ledger_date) AS ledger_date, + MIN(voucher_no) AS voucher_no, + MIN(compare_voucher_no) AS compare_voucher_no, + GROUP_CONCAT(DISTINCT status_key) AS source_statuses, + COUNT(*) AS source_group_count + FROM classified + WHERE COALESCE(identity_key, '') <> '' + GROUP BY identity_key + ) + SELECT + ?, ?, ?, identity_key, compare_voucher_no, + fiscal_year, ledger_date, voucher_no, + CASE final_rank {status_case} ELSE '' END, + final_rank, COALESCE(source_statuses, ''), source_group_count, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM resolved + WHERE final_rank < 99 + """, + (YEAR, YEAR, signature, YEAR, YEAR, signature), + ) + return int( + conn.execute( + """ + SELECT COUNT(*) + FROM wehago_compare_final_status_projection + WHERE start_year = ? AND end_year = ? AND signature = ? + """, + (YEAR, YEAR, signature), + ).fetchone()[0] + or 0 + ) + + +def clear_projection_caches(conn: sqlite3.Connection, signature: str) -> None: + for table_name in ( + "wehago_compare_query_page_cache", + "wehago_metric_count_cache", + "wehago_summary_range_cache", + ): + conn.execute( + f""" + DELETE FROM {table_name} + WHERE start_year = ? AND end_year = ? + AND signature = ? + """, + (YEAR, YEAR, signature), + ) + + def main() -> None: global MANUAL_OFFSET_EXCEPTED_IDENTITIES, RAW_ERP_ROWS_BY_DRAFT_BASE, YEAR parser = argparse.ArgumentParser(description="Reconcile a WEHAGO comparison query projection to DB rows.") @@ -1318,13 +2877,20 @@ def main() -> None: conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row cur = conn.cursor() + log_step(f"selecting source projection for {YEAR}") source_signature = latest_projection_signature(cur) - reconciled_prefix = f"{QUERY_PROJECTION_VERSION}|db-reconciled-v1|" + reconciled_prefix = f"{QUERY_PROJECTION_VERSION}|{RECONCILED_PROJECTION_VERSION}|" target_signature = source_signature if source_signature.startswith(reconciled_prefix) else f"{reconciled_prefix}{source_signature}" + log_step(f"source={source_signature[:120]} target={target_signature[:120]}") + log_step("loading ERP source rows") RAW_ERP_ROWS_BY_DRAFT_BASE = load_raw_erp_rows_by_draft_base(conn) + log_step("loading manual excepted identities") MANUAL_OFFSET_EXCEPTED_IDENTITIES = load_manual_offset_excepted_identities(conn) + log_step("loading WEHAGO comparison rows") db_wehago = load_db_wehago(conn) + log_step(f"loading query groups for source signature ({len(db_wehago)} WEHAGO vouchers)") groups = load_groups(conn, source_signature) + log_step("loading raw WEHAGO ledger rows") ledger_rows_by_key: dict[str, list[sqlite3.Row]] = defaultdict(list) for row in conn.execute( @@ -1341,6 +2907,7 @@ def main() -> None: selected_by_key: dict[str, tuple[str, dict[str, Any]]] = {} duplicate_counter = 0 removed_non_db = 0 + log_step("selecting best WEHAGO groups") for status_key in WEHAGO_STATUSES: for group in groups.get(status_key, []): for identity, split_group in split_group_by_wehago_voucher(group, status_key).items(): @@ -1360,6 +2927,7 @@ def main() -> None: duplicate_counter += 1 missing_db = sorted(set(db_wehago) - set(selected_by_key)) + log_step(f"adding DB-only WEHAGO groups: {len(missing_db)}") for key in missing_db: selected_by_key[key] = ( "voucher_unmatched", @@ -1367,6 +2935,7 @@ def main() -> None: ) status_groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + log_step("supplementing raw WEHAGO rows and normalizing statuses") for key in sorted(selected_by_key): status_key, group = selected_by_key[key] group = supplement_group_with_missing_wehago_rows( @@ -1375,9 +2944,25 @@ def main() -> None: key, ledger_rows_by_key.get(key, []), ) - status_groups[status_key].append(group) + final_status_key = normalized_wehago_status(status_key, group) + status_groups[final_status_key].append(retag_group(group, final_status_key)) + log_step("applying excepted rules") move_excepted_groups(status_groups) + log_step("reapplying split revenue, cancel/reissue, and exact reversal rules") + apply_split_revenue_matches_to_status_groups(status_groups) + apply_cancel_reissue_final_match(status_groups) + apply_accrual_principle_matches(status_groups, groups) + apply_split_draft_row_matches(status_groups) + apply_exact_reversal_pairs_to_status_groups(status_groups) + log_step("enforcing final WEHAGO status invariants") + invariant_diagnostics = enforce_wehago_status_invariants(status_groups) + log_step("rechecking cancel/reissue and exact reversal rules after invariants") + apply_cancel_reissue_final_match(status_groups) + apply_accrual_principle_matches(status_groups, groups) + apply_split_draft_row_matches(status_groups) + apply_exact_reversal_pairs_to_status_groups(status_groups) + invariant_diagnostics = enforce_wehago_status_invariants(status_groups) excepted_wehago_keys = { group_identity(group) for group in status_groups.get("voucher_excepted") or [] @@ -1390,6 +2975,7 @@ def main() -> None: if key } for status_key in ERP_STATUSES: + log_step(f"normalizing ERP side groups: {status_key}") seen_erp: set[str] = set() for group in groups.get(status_key, []): cleaned = clean_group_rows(group, status_key) @@ -1419,6 +3005,17 @@ def main() -> None: ) cleaned["summary"] = rebuild_summary(cleaned, final_status_key) else: + if group_identity(cleaned) in excepted_wehago_keys: + continue + normalized_rows = [ + row + for row in cleaned.get("rows") or [] + if not (has_wehago_value(row) and row_wehago_identity(row, cleaned) in excepted_wehago_keys) + ] + if not normalized_rows: + continue + cleaned["rows"] = normalized_rows + cleaned["summary"] = rebuild_summary(cleaned, status_key) final_status_key = status_key summary = cleaned.get("summary") or {} identity = "|".join( @@ -1431,6 +3028,7 @@ def main() -> None: seen_erp.add(identity) status_groups[final_status_key].append(cleaned) + log_step("writing reconciled projection") conn.execute("BEGIN") try: conn.execute( @@ -1441,11 +3039,20 @@ def main() -> None: "DELETE FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ?", (YEAR, YEAR, target_signature), ) + clear_projection_caches(conn, target_signature) counts: dict[str, int] = {} for status_key in ALL_VOUCHER_STATUSES: + log_step(f"inserting {status_key}: {len(status_groups.get(status_key, []))}") for group_index, group in enumerate(status_groups.get(status_key, []), start=1): insert_group(conn, target_signature, status_key, group_index, group) counts[status_key] = len(status_groups.get(status_key, [])) + log_step("storing query metric projection") + store_query_metric_projection(conn, target_signature, counts) + log_step("backfilling final status projection") + final_projection_rows = backfill_final_status_projection(conn, target_signature) + log_step("activating projection signature") + activate_projection_signature(conn, target_signature, counts, invariant_diagnostics) + log_step("updating snapshot row counts") update_snapshot_row_counts(conn, counts) conn.execute( """ @@ -1464,6 +3071,8 @@ def main() -> None: "removed_duplicate_groups": duplicate_counter, "removed_non_db_groups": removed_non_db, "added_db_only_groups": len(missing_db), + "final_projection_rows": final_projection_rows, + "invariant_diagnostics": invariant_diagnostics, "created_at": datetime.now().isoformat(timespec="seconds"), }, ensure_ascii=False, @@ -1471,6 +3080,7 @@ def main() -> None: ), ) conn.commit() + log_step("commit complete") except Exception: conn.rollback() raise @@ -1485,6 +3095,8 @@ def main() -> None: "removed_duplicate_groups": duplicate_counter, "removed_non_db_groups": removed_non_db, "added_db_only_groups": len(missing_db), + "final_projection_rows": final_projection_rows, + "invariant_diagnostics": invariant_diagnostics, }, ensure_ascii=False, ) diff --git a/scripts/run_wehago_logic_iteration.py b/scripts/run_wehago_logic_iteration.py new file mode 100644 index 0000000..ec73f36 --- /dev/null +++ b/scripts/run_wehago_logic_iteration.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import argparse +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from runtime_config import DB_PATH + + +DEFAULT_SAMPLES = ( + "2025-02-25-00029", + "2025-01-10-00006", + "2025-01-15-50027", + "2025-01-15-50028", + "2025-01-21-50197", + "2025-01-21-50198", +) + + +def run_command(command: list[str]) -> str: + proc = subprocess.run(command, cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True) + if proc.returncode != 0: + payload = { + "command": command, + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + raise SystemExit(json.dumps(payload, ensure_ascii=False, indent=2)) + return proc.stdout + + +def parse_sample(value: str, default_year: int) -> tuple[int, str, str]: + text = value.strip() + parts = text.replace("/", "-").split("-") + if len(parts) == 4: + year, month, day, voucher = parts + elif len(parts) == 3: + year = str(default_year) + month, day, voucher = parts + else: + raise ValueError(f"Invalid sample format: {value}") + return int(year), f"{int(month):02d}-{int(day):02d}", f"{int(voucher):05d}" if voucher.isdigit() else voucher + + +def active_signature(conn: sqlite3.Connection, year: int) -> str: + row = conn.execute( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = ? + LIMIT 1 + """, + (f"wehago_active_query_projection:{year}:{year}",), + ).fetchone() + if not row: + return "" + try: + payload = json.loads(row[0] or "{}") + except Exception: + return "" + return str(payload.get("signature") or "") if isinstance(payload, dict) else "" + + +def validate_projection(conn: sqlite3.Connection, year: int, signature: str) -> dict[str, Any]: + counts = { + row[0]: int(row[1] or 0) + for row in conn.execute( + """ + SELECT status_key, COUNT(*) + FROM wehago_compare_query_groups + WHERE start_year = ? AND end_year = ? AND signature = ? + AND status_key IN ('voucher_matched', 'voucher_unmatched', 'voucher_recheck', 'voucher_excepted') + GROUP BY status_key + """, + (year, year, signature), + ).fetchall() + } + final_counts = { + row[0]: int(row[1] or 0) + for row in conn.execute( + """ + SELECT final_status, COUNT(*) + FROM wehago_compare_final_status_projection + WHERE start_year = ? AND end_year = ? AND signature = ? + GROUP BY final_status + """, + (year, year, signature), + ).fetchall() + } + raw_total = int( + conn.execute( + """ + SELECT COUNT(DISTINCT compare_voucher_no) + FROM wehago_ledger_rows + WHERE fiscal_year = ? + AND COALESCE(compare_voucher_no, '') <> '' + """, + (year,), + ).fetchone()[0] + or 0 + ) + recheck_without_erp = int( + conn.execute( + """ + SELECT COUNT(*) + FROM wehago_compare_query_groups g + WHERE g.start_year = ? AND g.end_year = ? AND g.signature = ? + AND g.status_key = 'voucher_recheck' + AND NOT EXISTS ( + SELECT 1 + FROM wehago_compare_query_rows r + WHERE r.start_year = g.start_year + AND r.end_year = g.end_year + AND r.signature = g.signature + AND r.status_key = g.status_key + AND r.group_index = g.group_index + AND COALESCE(r.voucher_account_name, '') <> '' + AND ( + ABS(COALESCE(r.voucher_debit, 0)) > 0.0001 + OR ABS(COALESCE(r.voucher_credit, 0)) > 0.0001 + ) + ) + """, + (year, year, signature), + ).fetchone()[0] + or 0 + ) + unmatched_with_erp = int( + conn.execute( + """ + SELECT COUNT(*) + FROM wehago_compare_query_groups g + WHERE g.start_year = ? AND g.end_year = ? AND g.signature = ? + AND g.status_key = 'voucher_unmatched' + AND EXISTS ( + SELECT 1 + FROM wehago_compare_query_rows r + WHERE r.start_year = g.start_year + AND r.end_year = g.end_year + AND r.signature = g.signature + AND r.status_key = g.status_key + AND r.group_index = g.group_index + AND COALESCE(r.voucher_account_name, '') <> '' + AND ( + ABS(COALESCE(r.voucher_debit, 0)) > 0.0001 + OR ABS(COALESCE(r.voucher_credit, 0)) > 0.0001 + ) + ) + """, + (year, year, signature), + ).fetchone()[0] + or 0 + ) + return { + "raw_total": raw_total, + "counts": counts, + "final_counts": final_counts, + "classified_total": sum(counts.values()), + "difference": raw_total - sum(counts.values()), + "recheck_without_erp": recheck_without_erp, + "unmatched_with_erp": unmatched_with_erp, + } + + +def sample_statuses(conn: sqlite3.Connection, year: int, signature: str, samples: list[str]) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for sample in samples: + sample_year, ledger_date, voucher_no = parse_sample(sample, year) + rows = [ + { + "status_key": row[0], + "ledger_date": row[1], + "voucher_no": row[2], + "draft_no": row[3], + "review_reason": row[4], + } + for row in conn.execute( + """ + SELECT status_key, ledger_date, voucher_no, draft_no, review_reason + FROM wehago_compare_query_groups + WHERE start_year = ? AND end_year = ? AND signature = ? + AND fiscal_year = ? + AND ledger_date = ? + AND voucher_no = ? + ORDER BY status_key, group_index + """, + (year, year, signature, sample_year, ledger_date, voucher_no), + ).fetchall() + ] + result.append({"sample": sample, "rows": rows}) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run a fast WEHAGO logic iteration: compile, reconcile, validate, sample-check, optionally prune.") + parser.add_argument("--year", type=int, default=2025) + parser.add_argument("--sample", action="append", default=[]) + parser.add_argument("--skip-compile", action="store_true") + parser.add_argument("--skip-reconcile", action="store_true") + parser.add_argument("--prune", action="store_true") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + python = str(repo_root / ".venv" / "bin" / "python") + if not args.skip_compile: + run_command( + [ + python, + "-m", + "py_compile", + "scripts/reconcile_wehago_projection_to_db.py", + "scripts/prune_wehago_projection_history.py", + "wehago_compare.py", + "main.py", + ] + ) + reconcile_output = "" + if not args.skip_reconcile: + reconcile_output = run_command([python, "scripts/reconcile_wehago_projection_to_db.py", "--year", str(args.year)]) + prune_output = "" + if args.prune: + prune_output = run_command( + [ + python, + "scripts/prune_wehago_projection_history.py", + "--start-year", + str(args.year), + "--end-year", + str(args.year), + "--execute", + ] + ) + + conn = sqlite3.connect(DB_PATH) + try: + signature = active_signature(conn, args.year) + samples = args.sample or list(DEFAULT_SAMPLES) + payload = { + "year": args.year, + "active_signature": signature, + "validation": validate_projection(conn, args.year, signature) if signature else {}, + "samples": sample_statuses(conn, args.year, signature, samples) if signature else [], + "reconcile_output_tail": reconcile_output.strip().splitlines()[-8:], + "prune_output": json.loads(prune_output) if prune_output.strip().startswith("{") else prune_output, + } + finally: + conn.close() + print(json.dumps(payload, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/templates/admin_users.html b/templates/admin_users.html index ae6d32c..afb6db0 100644 --- a/templates/admin_users.html +++ b/templates/admin_users.html @@ -121,6 +121,104 @@ color: var(--ink); } + .last-login-button { + border: 0; + background: transparent; + box-shadow: none; + padding: 0; + min-height: 0; + color: var(--ink); + font: inherit; + font-weight: 700; + line-height: 1.25; + text-align: left; + text-decoration: none; + cursor: pointer; + } + + .last-login-button:hover, + .last-login-button:focus { + background: transparent; + box-shadow: none; + text-decoration: none; + } + + .login-history-modal { + position: fixed; + inset: 0; + z-index: 80; + display: none; + align-items: center; + justify-content: center; + padding: 24px; + background: rgba(17, 24, 39, 0.28); + } + + .login-history-modal.open { + display: flex; + } + + .login-history-dialog { + width: min(1080px, calc(100vw - 48px)); + max-height: min(760px, calc(100vh - 48px)); + overflow: auto; + border: 1px solid var(--line); + border-radius: 0; + background: #ffffff; + box-shadow: none; + padding: 18px; + } + + .login-history-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 12px; + } + + .login-history-head strong { + font-size: 18px; + line-height: 1.3; + } + + .login-history-close { + border: 0; + background: transparent; + box-shadow: none; + color: var(--ink); + padding: 0; + min-height: 0; + font-weight: 800; + cursor: pointer; + } + + .login-history-close:hover, + .login-history-close:focus { + background: transparent; + box-shadow: none; + } + + .login-history-table-wrap { + overflow: auto; + } + + .login-history-table { + width: 100%; + table-layout: fixed; + } + + .login-history-table th, + .login-history-table td { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .login-history-table .ua-cell { + max-width: 360px; + } + @media (max-width: 1100px) { .admin-user-main-row { grid-template-columns: repeat(2, minmax(0, 1fr)); @@ -201,12 +299,14 @@ 페이지 권한 상태 관리자 + 최종접속기록(KST) 수정일 {% for user in users %} {{ '활성' if user.is_active else '비활성' }} {{ '예' if user.is_admin else '-' }} + + {% if user.last_login_at %} + + {% else %} + - + {% endif %} + {{ user.updated_at }} {% else %} - 사용자가 없습니다. + 사용자가 없습니다. {% endfor %} + + {% endblock %} {% block script %} @@ -253,6 +374,79 @@ const role = form.querySelector("[name='role']"); const active = form.querySelector("[name='is_active']"); const permissionInputs = [...form.querySelectorAll("[name='permissions']")]; + const loginHistoryModal = document.getElementById("loginHistoryModal"); + const loginHistoryTitle = document.getElementById("loginHistoryTitle"); + const loginHistoryBody = document.getElementById("loginHistoryBody"); + const loginHistoryClose = document.getElementById("loginHistoryClose"); + + function escapeHtml(value) { + return String(value ?? "") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + + function openLoginHistoryModal() { + loginHistoryModal?.classList.add("open"); + loginHistoryModal?.setAttribute("aria-hidden", "false"); + } + + function closeLoginHistoryModal() { + loginHistoryModal?.classList.remove("open"); + loginHistoryModal?.setAttribute("aria-hidden", "true"); + } + + function renderLoginHistory(payload) { + const events = Array.isArray(payload?.events) ? payload.events : []; + if (!events.length) { + loginHistoryBody.innerHTML = '
접속이력이 없습니다.
'; + return; + } + loginHistoryBody.innerHTML = ` + + + + + + + + + + + + + ${events.map((event) => ` + + + + + + + + + `).join("")} + +
일시(KST)결과아이디IP실패사유사용환경
${escapeHtml(event.created_at_kst || event.created_at)}${event.success ? "성공" : "실패"}${escapeHtml(event.username || "-")}${escapeHtml(event.ip_address || "-")}${escapeHtml(event.failure_reason || "-")}${escapeHtml(event.user_agent || "-")}
+ `; + } + + async function loadLoginHistory(userId, label) { + loginHistoryTitle.textContent = `${label || "사용자"} 접속이력`; + loginHistoryBody.innerHTML = '
접속이력을 불러오는 중입니다.
'; + openLoginHistoryModal(); + try { + const response = await fetch(`/admin/users/${encodeURIComponent(userId)}/login-events?limit=100`, { + headers: { Accept: "application/json" }, + }); + const payload = await response.json(); + if (!response.ok) throw new Error(payload?.error || "접속이력을 불러오지 못했습니다."); + renderLoginHistory(payload); + } catch (error) { + loginHistoryBody.innerHTML = `
${escapeHtml(error.message || "접속이력을 불러오지 못했습니다.")}
`; + } + } document.querySelectorAll(".user-edit-button").forEach((button) => { button.addEventListener("click", () => { @@ -270,6 +464,23 @@ }); }); + document.querySelectorAll(".last-login-button").forEach((button) => { + button.addEventListener("click", () => { + const row = button.closest("tr"); + loadLoginHistory(button.dataset.loginHistoryUser, row?.dataset.username || button.textContent.trim()); + }); + }); + + loginHistoryClose?.addEventListener("click", closeLoginHistoryModal); + loginHistoryModal?.addEventListener("click", (event) => { + if (event.target === loginHistoryModal) closeLoginHistoryModal(); + }); + document.addEventListener("keydown", (event) => { + if (event.key === "Escape" && loginHistoryModal?.classList.contains("open")) { + closeLoginHistoryModal(); + } + }); + role.addEventListener("change", () => { const isAdmin = role.value === "admin"; permissionInputs.forEach((input) => { diff --git a/templates/base.html b/templates/base.html index 3a55f35..fae1270 100644 --- a/templates/base.html +++ b/templates/base.html @@ -82,28 +82,34 @@ .nav { display: flex; - gap: 10px; - flex-wrap: wrap; + gap: 7px; + flex-wrap: nowrap; align-items: center; background: rgba(255, 255, 255, 0.9); border: 1px solid var(--line); border-radius: 18px; - padding: 8px; + padding: 7px 8px; box-shadow: 0 10px 24px rgba(21, 24, 29, 0.06); + overflow-x: auto; + overflow-y: hidden; + scrollbar-width: thin; } .nav-spacer { flex: 1 1 auto; - min-width: 12px; + min-width: 6px; } .nav a { + flex: 0 0 auto; text-decoration: none; color: var(--ink); - padding: 9px 13px; + padding: 8px 10px; border-radius: 10px; font-weight: 700; - font-size: 14px; + font-size: 13px; + line-height: 1.15; + white-space: nowrap; transition: background 0.18s ease, color 0.18s ease, border-color 0.18s ease; } @@ -519,8 +525,9 @@ } .sync-status { + flex: 0 0 auto; margin-left: auto; - max-width: 320px; + max-width: 230px; border: 1px solid var(--line); border-radius: 999px; background: rgba(248, 250, 252, 0.92); @@ -535,10 +542,11 @@ .view-mode-switch { position: relative; + flex: 0 0 auto; display: inline-flex; align-items: center; justify-content: center; - width: 80px; + width: 72px; height: var(--status-widget-height); min-height: var(--status-widget-height); border-radius: 999px; @@ -589,11 +597,11 @@ } .view-mode-switch[data-mode="dual"]::before { - transform: translateX(38px); + transform: translateX(30px); } .view-mode-switch[data-mode="dual"] .label { - left: 41px; + left: 33px; } .view-mode-switch[data-mode="single"]::before { diff --git a/templates/cost_analysis.html b/templates/cost_analysis.html index 7fbf88e..06b6c4c 100644 --- a/templates/cost_analysis.html +++ b/templates/cost_analysis.html @@ -53,8 +53,8 @@ .cost-kpis { display: grid; - grid-template-columns: repeat(10, minmax(0, 1fr)); - gap: 8px; + grid-template-columns: repeat(9, minmax(0, 1fr)); + gap: 6px; } .cost-kpi { @@ -69,6 +69,9 @@ font-size: 11px; font-weight: 800; margin-bottom: 5px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; } .cost-kpi strong { @@ -77,6 +80,8 @@ line-height: 1.15; white-space: nowrap; word-break: keep-all; + overflow: hidden; + text-overflow: ellipsis; } .summary-accordion { @@ -174,7 +179,8 @@ } .cost-table { - min-width: 2820px; + width: 100%; + min-width: 0; table-layout: fixed; border-collapse: separate; border-spacing: 0; @@ -182,8 +188,8 @@ .cost-table th, .cost-table td { - padding: 8px 9px; - font-size: 12px; + padding: 7px 5px; + font-size: 11px; vertical-align: middle; border-right: 1px solid #e6e8ec; background: #ffffff; @@ -238,29 +244,50 @@ position: sticky; left: 0; z-index: 6; + background: #ffffff; + background-clip: padding-box; box-shadow: 1px 0 0 #d9dde3; } .sticky-col-2 { position: sticky; - left: 130px; + left: 94px; z-index: 6; + background: #ffffff; + background-clip: padding-box; box-shadow: 1px 0 0 #d9dde3; } .sticky-col-3 { position: sticky; - left: 236px; + left: 172px; z-index: 6; + background: #ffffff; + background-clip: padding-box; box-shadow: 1px 0 0 #d9dde3; } thead .sticky-col, thead .sticky-col-2, thead .sticky-col-3 { + background: #edf4f4; z-index: 70 !important; } + .cost-table tbody td.sticky-col, + .cost-table tbody td.sticky-col-2, + .cost-table tbody td.sticky-col-3 { + background: #ffffff; + background-clip: padding-box; + z-index: 16; + } + + .cost-table tbody tr:hover td.sticky-col, + .cost-table tbody tr:hover td.sticky-col-2, + .cost-table tbody tr:hover td.sticky-col-3 { + background: #f8fbfb; + } + .summary-table-body td { background: #fbfcfd; font-weight: 900; @@ -308,12 +335,15 @@ .summary-table-body .sticky-col-3 { z-index: 8; background: #fbfcfd; + background-clip: padding-box; } .summary-table-body.open .summary-data-row .sticky-col, .summary-table-body.open .summary-data-row .sticky-col-2, .summary-table-body.open .summary-data-row .sticky-col-3 { z-index: 58; + background: #fbfcfd; + background-clip: padding-box; } .summary-row-toggle { @@ -450,13 +480,13 @@ box-shadow: 0 1px 2px rgba(15, 23, 42, 0.12); } - .col-pm { width: 130px; } - .col-code { width: 106px; } - .col-name { width: 240px; } - .col-small { width: 82px; } - .col-date { width: 104px; } - .col-money { width: 118px; text-align: right; } - .col-rate { width: 86px; text-align: right; } + .col-pm { width: 94px; } + .col-code { width: 78px; } + .col-name { width: 148px; } + .col-small { width: 54px; } + .col-date { width: 72px; } + .col-money { width: 72px; text-align: right; } + .col-rate { width: 62px; text-align: right; } .group-pre { background: #f3f8ee !important; } .group-during { background: #eef5fb !important; } @@ -475,7 +505,7 @@ background: transparent; color: #111827; box-shadow: none; - font-size: 12px; + font-size: 11px; font-weight: 800; text-align: right; } @@ -520,7 +550,7 @@ background: transparent; color: #0f766e; box-shadow: none; - font-size: 12px; + font-size: 11px; font-weight: 900; text-align: left; } @@ -588,7 +618,7 @@ @media (max-width: 720px) { .cost-kpis { overflow-x: auto; - grid-template-columns: repeat(10, minmax(96px, 1fr)); + grid-template-columns: repeat(9, minmax(96px, 1fr)); } .summary-grid { @@ -600,12 +630,6 @@ {% block content %}
-
-
-

프로젝트 손익분석

-
-
-
@@ -681,7 +705,7 @@ 사업전 사업중 사업후 - 총계 + 총계 원가 @@ -698,6 +722,7 @@ + @@ -718,7 +743,7 @@ - +
@@ -897,14 +922,13 @@ const items = [ ["프로젝트", `${formatNumber(summary.project_count)}건`], ["도급금액", formatNumber(summary.contract_amount)], - ["청구금액", formatNumber(summary.billing_amount)], - ["청구수익", formatNumber(summary.revenue_amount)], - ["수금금액", formatNumber(summary.collection_amount)], + ["기간 청구금액", formatNumber(summary.billing_amount)], + ["기간 수금금액", formatNumber(summary.collection_amount)], ["잔여도급금액", formatBalance(summary.contract_balance_amount)], ["총비용", formatNumber(summary.total_cost)], ["이윤", formatNumber(summary.profit_amount)], - ["청구수익률", formatRate(summary.revenue_profit_rate)], ["수금수익률", formatRate(summary.collection_profit_rate)], + ["누적수익률", formatRate(summary.cumulative_profit_rate)], ]; kpis.innerHTML = items.map(([label, value]) => `
${escapeHtml(label)}${escapeHtml(value)}
`).join(""); } @@ -963,6 +987,7 @@ ${moneyCell(summary.profit_amount)} ${rateCell(summary.revenue_profit_rate)} ${rateCell(summary.collection_profit_rate)} + ${rateCell(summary.cumulative_profit_rate)} `; } @@ -1024,7 +1049,7 @@ function renderTable() { const rows = latestDisplayRows.length ? latestDisplayRows : getFilteredRows(); if (!rows.length) { - body.innerHTML = `조건에 맞는 데이터가 없습니다.`; + body.innerHTML = `조건에 맞는 데이터가 없습니다.`; return; } const viewportHeight = tableWrap?.clientHeight || 600; @@ -1035,8 +1060,8 @@ const topHeight = startIndex * virtualRowHeight; const bottomHeight = Math.max(0, (rows.length - endIndex) * virtualRowHeight); const visibleRows = rows.slice(startIndex, endIndex); - const topSpacer = topHeight ? `` : ""; - const bottomSpacer = bottomHeight ? `` : ""; + const topSpacer = topHeight ? `` : ""; + const bottomSpacer = bottomHeight ? `` : ""; body.innerHTML = topSpacer + visibleRows.map((row, visibleOffset) => { const rowIndex = startIndex + visibleOffset; const pre = row.phases?.pre || {}; @@ -1083,6 +1108,7 @@ ${formatNumber(row.profit_amount)} ${formatRate(row.revenue_profit_rate)} ${formatRate(row.collection_profit_rate)} + ${formatRate(row.cumulative_profit_rate)} `; }).join("") + bottomSpacer; @@ -1091,10 +1117,20 @@ function recalcVisibleSummary() { const rows = getFilteredRows(); latestDisplayRows = rows; - const summary = rows.reduce((acc, row) => { + const buildSummary = (usePeriodValues = false) => rows.reduce((acc, row) => { for (const key of ["contract_amount", "billing_amount", "collection_amount", "contract_balance_amount", "revenue_amount", "cost_total", "sga_total", "sales_total", "total_cost", "profit_amount"]) { acc[key] += Number(row[key] || 0); } + if (usePeriodValues) { + acc.billing_amount += Number(row.period_billing_amount || 0) - Number(row.billing_amount || 0); + acc.collection_amount += Number(row.period_collection_amount || 0) - Number(row.collection_amount || 0); + acc.revenue_amount += Number(row.period_revenue_amount || 0) - Number(row.revenue_amount || 0); + acc.cost_total += Number(row.period_cost_total || 0) - Number(row.cost_total || 0); + acc.sga_total += Number(row.period_sga_total || 0) - Number(row.sga_total || 0); + acc.sales_total += Number(row.period_sales_total || 0) - Number(row.sales_total || 0); + acc.total_cost += Number(row.period_total_cost || 0) - Number(row.total_cost || 0); + acc.profit_amount += Number(row.period_profit_amount || 0) - Number(row.profit_amount || 0); + } for (const phase of ["pre", "during", "post"]) { const bucket = row.phases?.[phase] || {}; for (const item of ["labor", "outsource", "overhead", "sga", "sales"]) { @@ -1115,17 +1151,26 @@ sales_total: 0, total_cost: 0, profit_amount: 0, + cumulative_profit_amount: 0, phases: { pre: { labor: 0, outsource: 0, overhead: 0, sga: 0, sales: 0 }, during: { labor: 0, outsource: 0, overhead: 0, sga: 0, sales: 0 }, post: { labor: 0, outsource: 0, overhead: 0, sga: 0, sales: 0 }, }, }); + const summary = buildSummary(false); summary.collection_rate = summary.contract_amount ? summary.collection_amount / summary.contract_amount * 100 : 0; summary.contract_profit_rate = summary.contract_amount ? summary.profit_amount / summary.contract_amount * 100 : 0; summary.revenue_profit_rate = summary.revenue_amount ? summary.profit_amount / summary.revenue_amount * 100 : 0; summary.collection_profit_rate = summary.collection_amount ? summary.profit_amount / summary.collection_amount * 100 : 0; - renderKpis(summary); + summary.cumulative_profit_rate = summary.collection_amount ? (summary.collection_amount - summary.total_cost) / summary.collection_amount * 100 : 0; + const kpiSummary = buildSummary(true); + kpiSummary.collection_rate = kpiSummary.contract_amount ? kpiSummary.collection_amount / kpiSummary.contract_amount * 100 : 0; + kpiSummary.contract_profit_rate = kpiSummary.contract_amount ? kpiSummary.profit_amount / kpiSummary.contract_amount * 100 : 0; + kpiSummary.revenue_profit_rate = kpiSummary.revenue_amount ? kpiSummary.profit_amount / kpiSummary.revenue_amount * 100 : 0; + kpiSummary.collection_profit_rate = kpiSummary.collection_amount ? kpiSummary.profit_amount / kpiSummary.collection_amount * 100 : 0; + kpiSummary.cumulative_profit_rate = summary.cumulative_profit_rate; + renderKpis(kpiSummary); renderSummaryTable(summary); } @@ -1136,7 +1181,9 @@ const controller = activeLoadController; searchButton.disabled = true; searchButton.textContent = "조회 중"; - body.innerHTML = `조회 중입니다.`; + kpis.innerHTML = `
조회조회 중
`; + summaryTableRow.innerHTML = ""; + body.innerHTML = `조회 중입니다.`; try { const params = new URLSearchParams({ start_date: lockedStartDate, end_date: lockedEndDate, mode: currentMode }); const response = await fetch(`/cost-analysis/data?${params.toString()}`, { @@ -1365,7 +1412,7 @@ event.preventDefault(); rememberDateInputs(); loadData().catch((error) => { - body.innerHTML = `${escapeHtml(error.message)}`; + body.innerHTML = `${escapeHtml(error.message)}`; }); }); @@ -1376,7 +1423,7 @@ currentMode = button.dataset.mode || "individual"; modeToggle.querySelectorAll("[data-mode]").forEach((item) => item.classList.toggle("active", item === button)); loadData().catch((error) => { - body.innerHTML = `${escapeHtml(error.message)}`; + body.innerHTML = `${escapeHtml(error.message)}`; }); }); @@ -1464,7 +1511,7 @@ }); loadData().catch((error) => { - body.innerHTML = `${escapeHtml(error.message)}`; + body.innerHTML = `${escapeHtml(error.message)}`; kpis.innerHTML = ""; summaryTableRow.innerHTML = ""; }); diff --git a/templates/login.html b/templates/login.html index 87361bc..9d6131d 100644 --- a/templates/login.html +++ b/templates/login.html @@ -81,7 +81,7 @@ -

한맥 인트라넷 로그인

+

한맥기술 ERP 점검

diff --git a/templates/wehago_benefit_entertainment.html b/templates/wehago_benefit_entertainment.html new file mode 100644 index 0000000..20f0843 --- /dev/null +++ b/templates/wehago_benefit_entertainment.html @@ -0,0 +1,921 @@ +{% extends "base.html" %} +{% block title %}복리/접대비 분류{% endblock %} + +{% block content %} +{% set filters = benefit_report.filters %} + + +
+
+
+

복리후생비/접대비 분류

+

{{ filters.start_year }}년부터 {{ filters.end_year }}년까지 WEHAGO 전표 기준

+
+
+ 엑셀 다운로드 +
+
+ + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + + + +
+
총 금액{{ "{:,.0f}".format(benefit_report.total_amount) }}
+
전표 행{{ "{:,}".format(benefit_report.row_count) }}
+
거래처 요약{{ "{:,}".format(benefit_report.vendor_summary_rows|length) }}
+
직급 확인{{ "{:,}".format(benefit_report.hanmac_grade_match_count) }}
+
미리보기{{ "{:,}".format(benefit_report.shown_count) }}
+
+ + 대체/원가/손익 {{ '포함' if filters.include_adjustments else '제외' }} + +
+
+ +
+

분류/연도별 합계

+
+ + + + + + {% for year in benefit_report.summary_years %} + + {% endfor %} + + + + + + {% for row in benefit_report.category_year_summary_rows %} + {% set is_drill_category = (row.account_group == '복리후생비' and row.category in ['임원 개인성 비용', '기타', '경조사']) or (row.account_group == '접대비' and row.category in ['기타', '경조사', '운동비']) %} + + + + {% for year in benefit_report.summary_years %} + + {% endfor %} + + + + {% else %} + + {% endfor %} + +
구분분류{{ year }}합계건수
{{ row.account_group }}{{ row.category }} + {% if is_drill_category and row.year_amounts.get(year, 0) %} + + {% else %} + {{ "{:,.0f}".format(row.year_amounts.get(year, 0)) }} + {% endif %} + + {% if is_drill_category and row.total_amount %} + + {% else %} + {{ "{:,.0f}".format(row.total_amount) }} + {% endif %} + + {% if is_drill_category and row.row_count %} + + {% else %} + {{ "{:,}".format(row.row_count) }} + {% endif %} +
조회된 자료가 없습니다.
+
+
+ +
+
+

거래처별 요약

+
+ + + + + + + + + + + + + + {% for row in benefit_report.vendor_summary_rows %} + + + + + + + + + + {% else %} + + {% endfor %} + +
구분분류거래처개인/귀속직급금액건수
{{ row.category }}{{ row.vendor_name }}{{ row.hanmac_member_grade or "" }}
조회된 자료가 없습니다.
+
+
+ +
+

전표 상세

+
+ + + + + + + + + + + + + + + + + + {% for row in benefit_report.detail_rows %} + + + + + + + + + + + + + + {% else %} + + {% endfor %} + +
연도일자전표번호계정거래처직급분류차변금액대변금액적요상대계정
{{ row.year }}{{ row.ledger_date }}{{ row.voucher_no }}{{ row.account_name }}{{ row.vendor_name }}{{ row.hanmac_member_grade or "" }} +
+ +
+ 		{{ "{:,.0f}".format(row.debit) }}{{ "{:,.0f}".format(row.credit) }}{{ row.description }}{{ row.counterpart_details }}
조회된 자료가 없습니다.
+
+
+ +
+

화면에는 {{ "{:,}".format(benefit_report.shown_count) }}행이 표시됩니다. 엑셀에는 현재 필터의 전체 {{ "{:,}".format(benefit_report.row_count) }}행이 포함됩니다.

+
+
+
+ + + +{% endblock %} + +{% block script %} + +{% endblock %} diff --git a/templates/wehago_compare.html b/templates/wehago_compare.html index b658910..88dd837 100644 --- a/templates/wehago_compare.html +++ b/templates/wehago_compare.html @@ -466,6 +466,12 @@ margin-left: 0; } + .status-card[data-count-pending="true"] .status-value { + color: #64748b; + font-size: 13px; + line-height: 1.15; + } + .status-card[data-status="matched"] { background: #f7faf8; border-color: #d2ddd6; @@ -1640,15 +1646,17 @@ class="status-card" data-target="detail-{{ section.key }}" data-status="{{ section.key }}" + data-count-pending="{% if wehago_compare.pending and section.count == 0 %}true{% else %}false{% endif %}" aria-expanded="false" > {{ section.label }} - {{ "{:,}".format(section.count) }} + + {% if wehago_compare.pending and section.count == 0 %}갱신 중{% else %}{{ "{:,}".format(section.count) }}{% endif %} + {% endif %} {% endfor %} -
{% for section in wehago_compare.metric_sections %} {% if section.key in ['voucher_matched', 'voucher_unmatched', 'voucher_recheck', 'voucher_excepted', 'hanmac_unconnected', 'erp_voucher_matched', 'erp_voucher_unmatched'] %} @@ -2024,12 +2032,22 @@ const getColumnClass = (field) => `col-${String(field || '').replace(/[^a-zA-Z0-9_]/g, '_')}`; - const renderVoucherDetailTable = (group) => { - if (!voucherDetailTableWrap) return; - const rows = Array.isArray(group?.rows) ? group.rows : []; - const lineColumns = [ + const getVoucherLineColumns = (statusKey = '') => { + if (statusKey === 'hanmac_unconnected') { + return [ + ['fiscal_year', '연도'], + ['proof_date', 'Hanmac 일자'], + ['voucher_no', '확정전표번호'], + ['draft_no', '가전표번호'], + ['voucher_account_name', 'Hanmac 계정'], + ['voucher_vendor', 'Hanmac 거래처'], + ['voucher_debit', 'Hanmac 차변'], + ['voucher_credit', 'Hanmac 대변'], + ['voucher_desc', 'Hanmac 적요'], + ]; + } + return [ ['fiscal_year', '연도'], - ['status_label', '구분'], ['ledger_date', 'WEHAGO 일자'], ['voucher_no', '전표번호'], ['draft_no', '가전표번호'], @@ -2044,6 +2062,16 @@ ['ledger_desc', 'WEHAGO 적요'], ['voucher_desc', 'ERP 적요'], ]; + }; + + const renderVoucherDetailTable = (group) => { + if (!voucherDetailTableWrap) return; + const rows = Array.isArray(group?.rows) ? group.rows : []; + const isHanmacUnconnected = String(group?.summary?.status_label || '').toLowerCase() === 'hanmac unconnected'; + const lineColumns = [ + ['status_label', '구분'], + ...getVoucherLineColumns(isHanmacUnconnected ? 'hanmac_unconnected' : ''), + ]; if (!rows.length) { voucherDetailTableWrap.innerHTML = '
표시할 상세 행이 없습니다.
'; return; @@ -2335,22 +2363,7 @@ }); }); }; - const lineColumns = [ - ['fiscal_year', '연도'], - ['ledger_date', 'WEHAGO 일자'], - ['voucher_no', '전표번호'], - ['draft_no', '가전표번호'], - ['ledger_account_name', 'WEHAGO 계정'], - ['voucher_account_name', 'ERP 계정'], - ['ledger_vendor', 'WEHAGO 거래처'], - ['voucher_vendor', 'ERP 거래처'], - ['ledger_debit', 'WEHAGO 차변'], - ['ledger_credit', 'WEHAGO 대변'], - ['voucher_debit', 'ERP 차변'], - ['voucher_credit', 'ERP 대변'], - ['ledger_desc', 'WEHAGO 적요'], - ['voucher_desc', 'ERP 적요'], - ]; + const lineColumns = getVoucherLineColumns(statusKey); const stripAccountCodePrefix = (value) => { const text = String(value ?? ''); return text @@ -2746,8 +2759,13 @@ sections.forEach((section) => { const card = document.querySelector(`.status-card[data-status="${escapeSelectorValue(section.key || '')}"]`); const valueNode = card?.querySelector('.status-value'); + const countValue = Number(section.count || 0); + const isPendingCount = Boolean(payload?.pending) && countValue === 0; + if (card) { + card.dataset.countPending = isPendingCount ? 'true' : 'false'; + } if (valueNode) { - valueNode.textContent = Number(section.count || 0).toLocaleString(); + valueNode.textContent = isPendingCount ? '갱신 중' : countValue.toLocaleString(); } }); const lastAction = payload?.last_action; diff --git a/wehago_compare.py b/wehago_compare.py index ff55abf..1584c83 100644 --- a/wehago_compare.py +++ b/wehago_compare.py @@ -4,6 +4,7 @@ import copy import hashlib from io import BytesIO import json +import logging import re import sqlite3 import subprocess @@ -24,6 +25,8 @@ from sqlalchemy import text from sqlalchemy.exc import OperationalError from runtime_config import WEHAGO_COMPARE_EXPORT_ROOT, WEHAGO_SOURCE_ROOT +logger = logging.getLogger(__name__) + VOUCHER_HEADERS = [ "결재상태", "가전표번호", @@ -79,7 +82,18 @@ STATUS_META = [ ("bridge_expense_review", "2단계 비교", "보통예금 출금을 미지급금 결제와 원 비용 전표까지 추적한 검토 후보"), ] -QUERY_PROJECTION_VERSION = "compare-query-v14-strict-erp-draft-year-scope" +QUERY_PROJECTION_VERSION = "compare-query-v17-hanmac-unconnected-source" +STATUS_PROJECTION_VERSION = "status-projection-v2" +HANMAC_UNCONNECTED_STATUS_PROJECTION_VERSION = f"{STATUS_PROJECTION_VERSION}-hanmac-unconnected-source" +EXPORT_COMPACT_STATUS_PROJECTION_VERSION = f"{STATUS_PROJECTION_VERSION}-export-cache" +COMPACT_EXPORT_STATUS_KEYS = { + "voucher_matched", + "erp_voucher_matched", + "voucher_unmatched", + "erp_voucher_unmatched", + "voucher_recheck", + "voucher_excepted", +} QUERY_VOUCHER_STATUS_KEYS = ( "voucher_matched", "voucher_unmatched", @@ -114,6 +128,18 @@ VOUCHER_GROUP_LINE_COLUMNS = [ ("voucher_desc", "ERP 적요"), ] +HANMAC_UNCONNECTED_LINE_COLUMNS = [ + ("fiscal_year", "연도"), + ("proof_date", "Hanmac 일자"), + ("voucher_no", "확정전표번호"), + ("draft_no", "가전표번호"), + ("voucher_account_name", "Hanmac 계정"), + ("voucher_vendor", "Hanmac 거래처"), + ("voucher_debit", "Hanmac 차변"), + ("voucher_credit", "Hanmac 대변"), + ("voucher_desc", "Hanmac 적요"), +] + DETAIL_COLUMN_MAP = { "matched": [ ("fiscal_year", "연도"), @@ -191,7 +217,7 @@ DETAIL_COLUMN_MAP = { "voucher_unmatched": VOUCHER_GROUP_LINE_COLUMNS, "voucher_recheck": VOUCHER_GROUP_LINE_COLUMNS, "voucher_excepted": VOUCHER_GROUP_LINE_COLUMNS, - "hanmac_unconnected": VOUCHER_GROUP_LINE_COLUMNS, + "hanmac_unconnected": HANMAC_UNCONNECTED_LINE_COLUMNS, "erp_voucher_unmatched": VOUCHER_GROUP_LINE_COLUMNS, "erp_voucher_matched": VOUCHER_GROUP_LINE_COLUMNS, } @@ -237,6 +263,8 @@ _STATUS_CACHE_WARMING: set[str] = set() _STATUS_CACHE_WARMING_LOCK = threading.Lock() _METRIC_COUNTS_WARMING: set[str] = set() _METRIC_COUNTS_WARMING_LOCK = threading.Lock() +_COMPARE_REFRESH_ENQUEUEING: set[str] = set() +_COMPARE_REFRESH_ENQUEUEING_LOCK = threading.Lock() _COMPARE_SNAPSHOT_JOB_EVENT = threading.Event() _COMPARE_SNAPSHOT_WORKER_LOCK = threading.Lock() _COMPARE_SNAPSHOT_WORKER_STARTED = False @@ -268,8 +296,9 @@ COMPARE_YEAR_REQUEUE_COOLDOWN_BACKFILL_SEC = 6 * 60 * 60 COMPARE_JOB_STALE_RUNNING_SEC = 3 * 60 * 60 ENABLE_COMPARE_QUERY_BACKGROUND_REBUILD = False ENABLE_GENERATED_RECHECK_CANDIDATES = False -RECHECK_RESOLUTION_POLICY_VERSION = "recheck-v31-strict-erp-draft-year-scope" -METRIC_COUNT_CACHE_VERSION = "voucher-summary-v13-strict-erp-draft-year-scope" +ENABLE_YEAR_SNAPSHOT_EXPORT_ROW_CACHE = False +RECHECK_RESOLUTION_POLICY_VERSION = "recheck-v33-adjacent-erp-auto-only" +METRIC_COUNT_CACHE_VERSION = "voucher-summary-v16-hanmac-unconnected-source" SUMMARY_CACHE_POLICY_VERSION = "dashboard-summary-v1-nonblocking" _PAIR_RECOMMEND_PERSIST_TTL_SEC = 1800 _PAIR_RECOMMEND_JOB_EVENT = threading.Event() @@ -465,6 +494,17 @@ def _compare_range_job_priority(start_year: int | None, end_year: int | None, to return 70 + min(span, 50) +def get_hanmac_erp_candidate_year_range( + start_year: int | None, + end_year: int | None, +) -> tuple[int | None, int | None]: + if start_year is None or end_year is None: + return None, None + low = min(int(start_year), int(end_year)) + high = max(int(start_year), int(end_year)) + return low - 1, high + 1 + + def _parse_db_timestamp(value: Any) -> datetime | None: text_value = clean(value) if not text_value: @@ -717,6 +757,361 @@ def _query_projection_signature(conn: Any, start_year: int | None, end_year: int ) +def _active_query_projection_signature( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> str: + if start_year is None or end_year is None: + return "" + try: + row = conn.execute( + text( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = :setting_key + LIMIT 1 + """ + ), + {"setting_key": f"wehago_active_query_projection:{int(start_year)}:{int(end_year)}"}, + ).first() + except Exception: + return "" + if not row or not row[0]: + return "" + try: + payload = json.loads(str(row[0] or "{}")) + except Exception: + return "" + if not isinstance(payload, dict): + return "" + signature = clean(payload.get("signature")) + if not signature: + return "" + if not signature.startswith(f"{QUERY_PROJECTION_VERSION}|"): + return "" + try: + exists = conn.execute( + text( + """ + SELECT 1 + FROM wehago_compare_query_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + LIMIT 1 + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ).first() + except Exception: + return "" + return signature if exists else "" + + +def _effective_query_projection_signature(conn: Any, start_year: int | None, end_year: int | None) -> str: + return _active_query_projection_signature(conn, start_year, end_year) or _query_projection_signature(conn, start_year, end_year) + + +def _status_projection_setting_key(status_key: str, start_year: int | None, end_year: int | None) -> str: + return f"wehago_active_status_projection:{clean(status_key)}:{int(start_year or 0)}:{int(end_year or 0)}" + + +def _hanmac_unconnected_status_projection_signature(conn: Any, start_year: int | None, end_year: int | None) -> str: + if start_year is None or end_year is None: + return "" + return "|".join( + [ + HANMAC_UNCONNECTED_STATUS_PROJECTION_VERSION, + "hanmac_unconnected", + str(int(start_year)), + str(int(end_year)), + _hanmac_unconnected_cache_key(conn, int(start_year), int(end_year)), + ] + ) + + +def _export_cache_status_projection_signature( + conn: Any, + status_key: str, + start_year: int | None, + end_year: int | None, +) -> str: + if start_year is None or end_year is None: + return "" + normalized_status = clean(status_key) + if normalized_status not in COMPACT_EXPORT_STATUS_KEYS: + return "" + signatures = [ + f"{year}:{_get_fast_year_export_row_cache_signature(conn, year)}" + for year in range(int(start_year), int(end_year) + 1) + ] + return "|".join( + [ + EXPORT_COMPACT_STATUS_PROJECTION_VERSION, + normalized_status, + str(int(start_year)), + str(int(end_year)), + *signatures, + ] + ) + + +def _status_projection_signature( + conn: Any, + status_key: str, + start_year: int | None, + end_year: int | None, +) -> str: + normalized_status = clean(status_key) + if normalized_status == "hanmac_unconnected": + return _hanmac_unconnected_status_projection_signature(conn, start_year, end_year) + if normalized_status in COMPACT_EXPORT_STATUS_KEYS: + return _export_cache_status_projection_signature(conn, normalized_status, start_year, end_year) + return "" + + +def _expected_status_projection_signatures_for_range( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> dict[str, str]: + if start_year is None or end_year is None: + return {} + try: + start = int(start_year) + end = int(end_year) + except (TypeError, ValueError): + return {} + if start > end: + start, end = end, start + year_signatures = [ + f"{year}:{_get_fast_year_export_row_cache_signature(conn, year)}" + for year in range(start, end + 1) + ] + hanmac_cache_key = "|".join( + [ + QUERY_PROJECTION_VERSION, + str(start), + str(end), + *year_signatures, + ] + ) + expected = { + "hanmac_unconnected": "|".join( + [ + HANMAC_UNCONNECTED_STATUS_PROJECTION_VERSION, + "hanmac_unconnected", + str(start), + str(end), + hanmac_cache_key, + ] + ) + } + for status_key in COMPACT_EXPORT_STATUS_KEYS: + expected[status_key] = "|".join( + [ + EXPORT_COMPACT_STATUS_PROJECTION_VERSION, + status_key, + str(start), + str(end), + *year_signatures, + ] + ) + return expected + + +def _active_status_projection_signatures_for_range( + conn: Any, + start_year: int | None, + end_year: int | None, + status_keys: Iterable[str] | None = None, +) -> dict[str, str]: + if start_year is None or end_year is None: + return {} + try: + start = int(start_year) + end = int(end_year) + except (TypeError, ValueError): + return {} + expected = _expected_status_projection_signatures_for_range(conn, start, end) + requested = [clean(status_key) for status_key in (status_keys or expected.keys())] + requested = [status_key for status_key in requested if status_key in expected] + if not requested: + return {} + setting_keys = { + _status_projection_setting_key(status_key, start, end): status_key + for status_key in requested + } + placeholders = ", ".join(f":key_{index}" for index, _key in enumerate(setting_keys)) + params = {f"key_{index}": key for index, key in enumerate(setting_keys)} + try: + rows = conn.execute( + text( + f""" + SELECT setting_key, setting_json + FROM wehago_compare_settings + WHERE setting_key IN ({placeholders}) + """ + ), + params, + ).mappings().all() + except Exception: + return {} + candidates: dict[str, str] = {} + for row in rows: + status_key = setting_keys.get(clean(row.get("setting_key"))) + if not status_key: + continue + try: + payload = json.loads(clean(row.get("setting_json")) or "{}") + except Exception: + continue + if not isinstance(payload, dict): + continue + signature = clean(payload.get("signature")) + if signature and signature == expected.get(status_key): + candidates[status_key] = signature + if not candidates: + return {} + status_placeholders = ", ".join(f":status_{index}" for index, _status in enumerate(candidates)) + exists_params = { + "start_year": start, + "end_year": end, + **{f"status_{index}": status for index, status in enumerate(candidates)}, + } + for index, (status_key, signature) in enumerate(candidates.items()): + exists_params[f"signature_{index}"] = signature + signature_conditions = " OR ".join( + f"(status_key = :status_{index} AND signature = :signature_{index})" + for index, _item in enumerate(candidates.items()) + ) + try: + existing_rows = conn.execute( + text( + f""" + SELECT status_key, signature + FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key IN ({status_placeholders}) + AND ({signature_conditions}) + GROUP BY status_key, signature + """ + ), + exists_params, + ).mappings().all() + except Exception: + return {} + existing = {(clean(row.get("status_key")), clean(row.get("signature"))) for row in existing_rows} + return { + status_key: signature + for status_key, signature in candidates.items() + if (status_key, signature) in existing + } + + +def _active_status_projection_signature( + conn: Any, + status_key: str, + start_year: int | None, + end_year: int | None, +) -> str: + if start_year is None or end_year is None: + return "" + normalized_status = clean(status_key) + expected_signature = _status_projection_signature(conn, normalized_status, start_year, end_year) + if not expected_signature: + return "" + try: + row = conn.execute( + text( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = :setting_key + LIMIT 1 + """ + ), + {"setting_key": _status_projection_setting_key(normalized_status, start_year, end_year)}, + ).first() + except Exception: + return "" + if not row or not row[0]: + return "" + try: + payload = json.loads(str(row[0] or "{}")) + except Exception: + return "" + if not isinstance(payload, dict): + return "" + signature = clean(payload.get("signature")) + if signature != expected_signature: + return "" + table_name = ( + "wehago_status_projection_groups" + if normalized_status == "hanmac_unconnected" or normalized_status in COMPACT_EXPORT_STATUS_KEYS + else "wehago_compare_query_groups" + ) + exists = conn.execute( + text( + f""" + SELECT 1 + FROM {table_name} + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + LIMIT 1 + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": signature, + }, + ).first() + return signature if exists else "" + + +def _store_active_status_projection_signature( + conn: Any, + status_key: str, + start_year: int | None, + end_year: int | None, + signature: str, + total_count: int, +) -> None: + if start_year is None or end_year is None or not clean(signature): + return + normalized_status = clean(status_key) + payload = { + "signature": clean(signature), + "status_key": normalized_status, + "start_year": int(start_year), + "end_year": int(end_year), + "total_count": int(total_count or 0), + "projection_version": STATUS_PROJECTION_VERSION, + } + conn.execute( + text( + """ + INSERT INTO wehago_compare_settings (setting_key, setting_json, updated_at) + VALUES (:setting_key, :setting_json, CURRENT_TIMESTAMP) + ON CONFLICT(setting_key) DO UPDATE SET + setting_json = excluded.setting_json, + updated_at = CURRENT_TIMESTAMP + """ + ), + { + "setting_key": _status_projection_setting_key(normalized_status, start_year, end_year), + "setting_json": json.dumps(payload, ensure_ascii=False), + }, + ) + + def _load_query_metric_projection( conn: Any, start_year: int | None, @@ -724,7 +1119,7 @@ def _load_query_metric_projection( ) -> dict[str, Any] | None: if start_year is None or end_year is None: return None - signature = _query_projection_signature(conn, start_year, end_year) + signature = _effective_query_projection_signature(conn, start_year, end_year) row = conn.execute( text( """ @@ -840,7 +1235,25 @@ def _load_broader_query_projection_counts_from_groups( except Exception: return None try: - current_signature_prefix = f"{QUERY_PROJECTION_VERSION}|%" + active_signature = "" + active_row = conn.execute( + """ + SELECT setting_json + FROM wehago_compare_settings + WHERE setting_key = ? + LIMIT 1 + """, + (f"wehago_active_query_projection:{int(start_year)}:{int(end_year)}",), + ).fetchone() + if active_row and active_row[0]: + try: + active_payload = json.loads(str(active_row[0] or "{}")) + if isinstance(active_payload, dict): + active_signature = clean(active_payload.get("signature")) + except Exception: + active_signature = "" + if not active_signature.startswith(f"{QUERY_PROJECTION_VERSION}|"): + return None scope_row = conn.execute( """ SELECT @@ -977,55 +1390,20 @@ def _fast_sqlite_query_group_page( signature, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups - WHERE start_year <= ? - AND end_year >= ? + WHERE start_year = ? + AND end_year = ? AND status_key = ? - AND signature LIKE ? + AND signature = ? GROUP BY start_year, end_year, signature - ORDER BY - CASE WHEN signature LIKE ? THEN 0 ELSE 1 END ASC, - CASE WHEN start_year = ? AND end_year = ? THEN 0 ELSE 1 END ASC, - (end_year - start_year) ASC, - max_updated_at DESC LIMIT 1 """, ( int(start_year), int(end_year), normalized_status, - current_signature_prefix, - current_signature_prefix, - int(start_year), - int(end_year), + active_signature, ), ).fetchone() - if not scope_row: - scope_row = conn.execute( - """ - SELECT - start_year, - end_year, - signature, - MAX(updated_at) AS max_updated_at - FROM wehago_compare_query_groups - WHERE start_year <= ? - AND end_year >= ? - AND status_key = ? - GROUP BY start_year, end_year, signature - ORDER BY - CASE WHEN start_year = ? AND end_year = ? THEN 0 ELSE 1 END ASC, - (end_year - start_year) ASC, - max_updated_at DESC - LIMIT 1 - """, - ( - int(start_year), - int(end_year), - normalized_status, - int(start_year), - int(end_year), - ), - ).fetchone() if not scope_row: return None projection_start_year = int(scope_row["start_year"] or 0) @@ -1274,6 +1652,835 @@ def _load_latest_metric_counts_cache_any_signature( return None +def _coerce_metric_counts_payload(payload: Any) -> dict[str, int] | None: + if not isinstance(payload, dict): + return None + return {status_key: int(payload.get(status_key, 0) or 0) for status_key, _, _ in STATUS_META} + + +def _load_latest_query_metric_projection_any_signature( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> dict[str, int] | None: + if start_year is None or end_year is None: + return None + signature = _query_projection_signature(conn, start_year, end_year) + row = conn.execute( + text( + """ + SELECT counts_json + FROM wehago_compare_query_metrics + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + LIMIT 1 + """ + ), + { + "start_year": start_year, + "end_year": end_year, + "signature": signature, + }, + ).first() + if not row or not row[0]: + return None + try: + return _coerce_metric_counts_payload(json.loads(str(row[0]))) + except Exception: + return None + + +def _load_latest_metric_counts_cache_any_signature_loose( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> dict[str, int] | None: + if start_year is None or end_year is None: + return None + current_signature = _metric_counts_signature(conn, start_year, end_year) + row = conn.execute( + text( + """ + SELECT signature, counts_json + FROM wehago_metric_count_cache + WHERE start_year = :start_year + AND end_year = :end_year + ORDER BY updated_at DESC, created_at DESC + LIMIT 1 + """ + ), + {"start_year": start_year, "end_year": end_year}, + ).first() + if not row or not row[1]: + return None + if clean(row[0]) != current_signature: + return None + try: + return _coerce_metric_counts_payload(json.loads(str(row[1]))) + except Exception: + return None + + +def _load_metric_counts_from_cached_subranges( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> dict[str, int] | None: + if start_year is None or end_year is None: + return None + selected_years = set(range(int(start_year), int(end_year) + 1)) + if not selected_years: + return None + candidates: list[dict[str, Any]] = [] + for table_name, signature_like in ( + ("wehago_compare_query_metrics", f"{QUERY_PROJECTION_VERSION}|%"), + ("wehago_metric_count_cache", None), + ): + rows = conn.execute( + text( + f""" + SELECT start_year, end_year, signature, counts_json, updated_at, created_at + FROM {table_name} + WHERE start_year >= :start_year + AND end_year <= :end_year + ORDER BY updated_at DESC, created_at DESC + """ + ), + {"start_year": start_year, "end_year": end_year}, + ).mappings().all() + for row in rows: + row_start = int(row.get("start_year") or 0) + row_end = int(row.get("end_year") or 0) + if row_start <= 0 or row_end < row_start: + continue + years = set(range(row_start, row_end + 1)) + if not years or not years.issubset(selected_years): + continue + try: + counts = _coerce_metric_counts_payload(json.loads(clean(row.get("counts_json")) or "{}")) + except Exception: + counts = None + if counts is None: + continue + signature = clean(row.get("signature")) + candidates.append( + { + "start_year": row_start, + "end_year": row_end, + "years": years, + "counts": counts, + "current_rank": 0 if signature_like and signature.startswith(QUERY_PROJECTION_VERSION) else 1, + "width": len(years), + "updated_at": clean(row.get("updated_at")) or clean(row.get("created_at")), + } + ) + if not candidates: + return None + candidates.sort(key=lambda item: (-int(item["width"]), int(item["current_rank"]), clean(item["updated_at"])), reverse=False) + used_years: set[int] = set() + merged = _empty_metric_counts() + for candidate in candidates: + years = set(candidate["years"]) + if years & used_years: + continue + used_years.update(years) + counts = dict(candidate["counts"]) + for status_key in merged: + merged[status_key] += int(counts.get(status_key, 0) or 0) + if used_years == selected_years: + break + return merged if used_years else None + + +WEHAGO_FINAL_STATUS_ORDER = ( + "voucher_matched", + "voucher_unmatched", + "voucher_recheck", + "voucher_excepted", +) +WEHAGO_FINAL_STATUS_PRIORITY = { + "voucher_excepted": 1, + "voucher_matched": 2, + "voucher_recheck": 3, + "voucher_unmatched": 4, +} + + +def _normalize_wehago_projection_date(value: Any, fiscal_year: int | None = None) -> str: + normalized = clean(value) + if not normalized: + return "" + if re.fullmatch(r"\d{4}-\d{2}-\d{2}", normalized): + return normalized + if re.fullmatch(r"\d{1,2}[-./]\d{1,2}", normalized) and fiscal_year: + month_text, day_text = re.split(r"[-./]", normalized) + return f"{int(fiscal_year):04d}-{int(month_text):02d}-{int(day_text):02d}" + matched = re.search(r"(20\d{2})[-./]?(\d{2})[-./]?(\d{2})", normalized) + if matched: + return f"{int(matched.group(1)):04d}-{int(matched.group(2)):02d}-{int(matched.group(3)):02d}" + return normalized + + +def _build_wehago_final_identity_from_summary(summary: dict[str, Any], fallback: str = "") -> dict[str, Any]: + fiscal_year = int(summary.get("fiscal_year") or 0) + voucher_no = clean(summary.get("voucher_no")) + ledger_date = _normalize_wehago_projection_date( + summary.get("ledger_date") or summary.get("proof_date"), + fiscal_year or None, + ) + first_voucher_no = clean(voucher_no.split(",", 1)[0]) if voucher_no else "" + compare_voucher_no = "" + if ledger_date and first_voucher_no: + compare_voucher_no = f"{ledger_date.replace('-', '')}-{first_voucher_no}" + identity_key = "|".join( + [ + str(fiscal_year), + ledger_date, + first_voucher_no or clean(fallback), + ] + ) + return { + "identity_key": identity_key, + "compare_voucher_no": compare_voucher_no, + "fiscal_year": fiscal_year, + "ledger_date": ledger_date, + "voucher_no": first_voucher_no, + } + + +def _store_wehago_final_status_projection( + conn: Any, + start_year: int | None, + end_year: int | None, + voucher_sections: dict[str, list[dict[str, Any]]], +) -> None: + if start_year is None or end_year is None: + return + signature = _effective_query_projection_signature(conn, start_year, end_year) + resolved: dict[str, dict[str, Any]] = {} + for status_key in WEHAGO_FINAL_STATUS_PRIORITY: + status_rank = int(WEHAGO_FINAL_STATUS_PRIORITY[status_key]) + for group_index, group in enumerate(list(voucher_sections.get(status_key) or [])): + summary = dict(group.get("summary") or {}) + identity = _build_wehago_final_identity_from_summary(summary, fallback=f"{status_key}:{group_index}") + identity_key = clean(identity.get("identity_key")) + if not identity_key: + continue + current = resolved.get(identity_key) + source_statuses = set((current or {}).get("source_statuses_set") or set()) + source_statuses.add(status_key) + source_group_count = int((current or {}).get("source_group_count") or 0) + 1 + if current is None or status_rank < int(current.get("final_rank") or 99): + resolved[identity_key] = { + **identity, + "final_status": status_key, + "final_rank": status_rank, + "source_statuses_set": source_statuses, + "source_group_count": source_group_count, + } + else: + current["source_statuses_set"] = source_statuses + current["source_group_count"] = source_group_count + try: + conn.execute( + text( + """ + DELETE FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ) + for row in resolved.values(): + source_statuses = sorted( + clean(value) + for value in (row.get("source_statuses_set") or set()) + if clean(value) + ) + conn.execute( + text( + """ + INSERT INTO wehago_compare_final_status_projection ( + start_year, end_year, signature, identity_key, compare_voucher_no, + fiscal_year, ledger_date, voucher_no, final_status, final_rank, + source_statuses, source_group_count, created_at, updated_at + ) VALUES ( + :start_year, :end_year, :signature, :identity_key, :compare_voucher_no, + :fiscal_year, :ledger_date, :voucher_no, :final_status, :final_rank, + :source_statuses, :source_group_count, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + "identity_key": clean(row.get("identity_key")), + "compare_voucher_no": clean(row.get("compare_voucher_no")), + "fiscal_year": int(row.get("fiscal_year") or 0), + "ledger_date": clean(row.get("ledger_date")), + "voucher_no": clean(row.get("voucher_no")), + "final_status": clean(row.get("final_status")), + "final_rank": int(row.get("final_rank") or 99), + "source_statuses": ",".join(source_statuses), + "source_group_count": int(row.get("source_group_count") or 0), + }, + ) + except OperationalError as exc: + logger.warning("Skipped final status projection write for %s-%s due to lock: %s", start_year, end_year, exc) + + +def backfill_wehago_final_status_projection_from_query_groups( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> int: + if start_year is None or end_year is None: + return 0 + signature = _load_latest_query_projection_signature_for_range(conn, start_year, end_year) + if not signature: + return 0 + priority_case = " ".join( + f"WHEN '{status_key}' THEN {priority}" + for status_key, priority in WEHAGO_FINAL_STATUS_PRIORITY.items() + ) + status_case = " ".join( + f"WHEN {priority} THEN '{status_key}'" + for status_key, priority in WEHAGO_FINAL_STATUS_PRIORITY.items() + ) + try: + conn.execute( + text( + """ + DELETE FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ) + conn.execute( + text( + f""" + INSERT INTO wehago_compare_final_status_projection ( + start_year, end_year, signature, identity_key, compare_voucher_no, + fiscal_year, ledger_date, voucher_no, final_status, final_rank, + source_statuses, source_group_count, created_at, updated_at + ) + WITH source_groups AS ( + SELECT + fiscal_year, + status_key, + CASE status_key {priority_case} ELSE 99 END AS status_rank, + COALESCE(NULLIF(TRIM(ledger_date), ''), NULLIF(TRIM(proof_date), ''), '') AS raw_date, + COALESCE(NULLIF(TRIM(voucher_no), ''), CAST(group_index AS TEXT)) AS raw_voucher_no + FROM wehago_compare_query_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + AND status_key IN ('voucher_matched', 'voucher_recheck', 'voucher_unmatched', 'voucher_excepted') + ), + normalized AS ( + SELECT + fiscal_year, + status_key, + status_rank, + CASE + WHEN raw_date GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' THEN raw_date + WHEN raw_date GLOB '[0-9][0-9]-[0-9][0-9]' THEN printf('%04d-%s', fiscal_year, raw_date) + WHEN raw_date GLOB '[0-9]-[0-9][0-9]' THEN printf('%04d-0%s', fiscal_year, raw_date) + ELSE raw_date + END AS ledger_date, + TRIM( + CASE + WHEN INSTR(raw_voucher_no, ',') > 0 THEN SUBSTR(raw_voucher_no, 1, INSTR(raw_voucher_no, ',') - 1) + ELSE raw_voucher_no + END + ) AS voucher_no + FROM source_groups + ), + classified AS ( + SELECT + status_key, + status_rank, + fiscal_year, + ledger_date, + voucher_no, + CAST(fiscal_year AS TEXT) || '|' || ledger_date || '|' || voucher_no AS identity_key, + CASE + WHEN ledger_date <> '' AND voucher_no <> '' THEN REPLACE(ledger_date, '-', '') || '-' || voucher_no + ELSE '' + END AS compare_voucher_no + FROM normalized + WHERE COALESCE(voucher_no, '') <> '' + ), + resolved AS ( + SELECT + identity_key, + MIN(status_rank) AS final_rank, + MIN(fiscal_year) AS fiscal_year, + MIN(ledger_date) AS ledger_date, + MIN(voucher_no) AS voucher_no, + MIN(compare_voucher_no) AS compare_voucher_no, + GROUP_CONCAT(DISTINCT status_key) AS source_statuses, + COUNT(*) AS source_group_count + FROM classified + WHERE COALESCE(identity_key, '') <> '' + GROUP BY identity_key + ) + SELECT + :start_year, + :end_year, + :signature, + identity_key, + compare_voucher_no, + fiscal_year, + ledger_date, + voucher_no, + CASE final_rank {status_case} ELSE '' END AS final_status, + final_rank, + COALESCE(source_statuses, ''), + source_group_count, + CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP + FROM resolved + WHERE final_rank < 99 + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ) + return int( + conn.execute( + text( + """ + SELECT COUNT(*) + FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ).scalar_one() + or 0 + ) + except OperationalError as exc: + logger.warning("Skipped final status projection backfill for %s-%s due to lock: %s", start_year, end_year, exc) + except Exception as exc: + logger.warning("Skipped final status projection backfill for %s-%s: %s", start_year, end_year, exc) + return 0 + + +def _load_latest_query_projection_signature_for_range( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> str: + if start_year is None or end_year is None: + return "" + active_signature = _active_query_projection_signature(conn, start_year, end_year) + if active_signature: + return active_signature + signature = _query_projection_signature(conn, start_year, end_year) + row = conn.execute( + text( + """ + SELECT signature + FROM wehago_compare_query_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + LIMIT 1 + """ + ), + {"start_year": start_year, "end_year": end_year, "signature": signature}, + ).first() + if row and row[0]: + return clean(row[0]) + row = conn.execute( + text( + """ + SELECT signature + FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + AND signature LIKE :signature_like + GROUP BY signature + ORDER BY COUNT(*) DESC, MAX(updated_at) DESC + LIMIT 1 + """ + ), + { + "start_year": start_year, + "end_year": end_year, + "signature_like": f"{QUERY_PROJECTION_VERSION}|%", + }, + ).first() + if row and row[0]: + return clean(row[0]) + row = conn.execute( + text( + """ + SELECT signature + FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + GROUP BY signature + ORDER BY MAX(updated_at) DESC, COUNT(*) DESC + LIMIT 1 + """ + ), + {"start_year": start_year, "end_year": end_year}, + ).first() + if row and row[0]: + return clean(row[0]) + return "" + + +def get_wehago_final_status_summary_from_conn( + conn: Any, + start_year: int | None, + end_year: int | None, + metric_counts: dict[str, int] | None = None, +) -> dict[str, Any]: + status_labels = { + "voucher_matched": "Voucher", + "voucher_recheck": "Recheck", + "voucher_unmatched": "Unmatched", + "voucher_excepted": "Excepted", + } + empty_counts = {status_key: 0 for status_key in WEHAGO_FINAL_STATUS_ORDER} + if start_year is None or end_year is None: + return { + "raw_total": 0, + "classified_total": 0, + "difference": 0, + "group_total": 0, + "group_difference": 0, + "counts": empty_counts, + "items": [ + {"key": key, "label": status_labels[key], "count": 0} + for key in WEHAGO_FINAL_STATUS_ORDER + ], + "source": "none", + } + raw_total = int( + conn.execute( + text( + """ + SELECT COUNT(DISTINCT compare_voucher_no) + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(compare_voucher_no, '') <> '' + """ + ), + {"start_year": int(start_year), "end_year": int(end_year)}, + ).scalar_one() + or 0 + ) + group_counts = { + key: int((metric_counts or {}).get(key, 0) or 0) + for key in WEHAGO_FINAL_STATUS_ORDER + } + final_counts = dict(empty_counts) + signature = _load_latest_query_projection_signature_for_range(conn, start_year, end_year) + if signature: + try: + projection_rows = conn.execute( + text( + """ + SELECT final_status, COUNT(*) AS voucher_count + FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + GROUP BY final_status + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + }, + ).mappings().all() + except OperationalError: + projection_rows = [] + if projection_rows: + universe_rows = conn.execute( + text( + """ + WITH raw_vouchers AS ( + SELECT fiscal_year, compare_voucher_no + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(compare_voucher_no, '') <> '' + GROUP BY fiscal_year, compare_voucher_no + ) + SELECT COALESCE(p.final_status, 'voucher_unmatched') AS final_status, COUNT(*) AS voucher_count + FROM raw_vouchers r + LEFT JOIN wehago_compare_final_status_projection p + ON p.start_year = :start_year + AND p.end_year = :end_year + AND p.signature = :signature + AND p.fiscal_year = r.fiscal_year + AND p.compare_voucher_no = r.compare_voucher_no + GROUP BY COALESCE(p.final_status, 'voucher_unmatched') + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + }, + ).mappings().all() + for row in universe_rows: + status_key = clean(row.get("final_status")) + if status_key in final_counts: + final_counts[status_key] = int(row.get("voucher_count") or 0) + group_rows = conn.execute( + text( + """ + SELECT final_status, SUM(source_group_count) AS group_count + FROM wehago_compare_final_status_projection + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + GROUP BY final_status + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + }, + ).mappings().all() + for row in group_rows: + status_key = clean(row.get("final_status")) + if status_key in group_counts: + group_counts[status_key] = int(row.get("group_count") or 0) + projection_outside_raw_total = int( + conn.execute( + text( + """ + WITH raw_vouchers AS ( + SELECT fiscal_year, compare_voucher_no + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(compare_voucher_no, '') <> '' + GROUP BY fiscal_year, compare_voucher_no + ) + SELECT COUNT(*) + FROM wehago_compare_final_status_projection p + LEFT JOIN raw_vouchers r + ON r.fiscal_year = p.fiscal_year + AND r.compare_voucher_no = p.compare_voucher_no + WHERE p.start_year = :start_year + AND p.end_year = :end_year + AND p.signature = :signature + AND ( + COALESCE(p.compare_voucher_no, '') = '' + OR r.compare_voucher_no IS NULL + ) + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + }, + ).scalar_one() + or 0 + ) + group_total = sum(group_counts.values()) + classified_total = sum(final_counts.values()) + return { + "raw_total": raw_total, + "classified_total": classified_total, + "difference": classified_total - raw_total, + "group_total": classified_total, + "group_difference": classified_total - raw_total, + "source_group_total": group_total, + "source_group_difference": group_total - raw_total, + "projection_outside_raw_total": projection_outside_raw_total, + "counts": final_counts, + "items": [ + {"key": key, "label": status_labels[key], "count": int(final_counts.get(key, 0) or 0)} + for key in WEHAGO_FINAL_STATUS_ORDER + ], + "source": "final_status_projection", + } + group_rows = conn.execute( + text( + """ + SELECT status_key, COUNT(*) AS group_count + FROM wehago_compare_query_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + AND status_key IN ('voucher_matched', 'voucher_unmatched', 'voucher_recheck', 'voucher_excepted') + GROUP BY status_key + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + }, + ).mappings().all() + for row in group_rows: + status_key = clean(row.get("status_key")) + if status_key in group_counts: + group_counts[status_key] = int(row.get("group_count") or 0) + priority_case = " ".join( + f"WHEN '{status_key}' THEN {priority}" + for status_key, priority in WEHAGO_FINAL_STATUS_PRIORITY.items() + ) + rows = conn.execute( + text( + f""" + WITH raw_vouchers AS ( + SELECT fiscal_year, compare_voucher_no + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(compare_voucher_no, '') <> '' + GROUP BY fiscal_year, compare_voucher_no + ), + classified AS ( + SELECT + CASE status_key {priority_case} ELSE 99 END AS status_rank, + fiscal_year, + CASE + WHEN COALESCE(NULLIF(ledger_date, ''), NULLIF(proof_date, ''), '') <> '' + AND COALESCE(NULLIF(voucher_no, ''), '') <> '' + THEN REPLACE(COALESCE(NULLIF(ledger_date, ''), NULLIF(proof_date, ''), ''), '-', '') + || '-' + || TRIM( + CASE + WHEN INSTR(voucher_no, ',') > 0 THEN SUBSTR(voucher_no, 1, INSTR(voucher_no, ',') - 1) + ELSE voucher_no + END + ) + ELSE '' + END AS compare_voucher_no + FROM wehago_compare_query_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + AND status_key IN ('voucher_matched', 'voucher_recheck', 'voucher_unmatched', 'voucher_excepted') + ), + resolved AS ( + SELECT fiscal_year, compare_voucher_no, MIN(status_rank) AS final_rank + FROM classified + WHERE COALESCE(compare_voucher_no, '') <> '' + GROUP BY fiscal_year, compare_voucher_no + ) + SELECT COALESCE(resolved.final_rank, :unmatched_rank) AS final_rank, COUNT(*) AS voucher_count + FROM raw_vouchers + LEFT JOIN resolved + ON resolved.fiscal_year = raw_vouchers.fiscal_year + AND resolved.compare_voucher_no = raw_vouchers.compare_voucher_no + GROUP BY COALESCE(resolved.final_rank, :unmatched_rank) + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + "unmatched_rank": int(WEHAGO_FINAL_STATUS_PRIORITY["voucher_unmatched"]), + }, + ).mappings().all() + rank_to_status = {priority: key for key, priority in WEHAGO_FINAL_STATUS_PRIORITY.items()} + for row in rows: + status_key = rank_to_status.get(int(row.get("final_rank") or 0)) + if status_key: + final_counts[status_key] = int(row.get("voucher_count") or 0) + elif raw_total > 0: + final_counts["voucher_unmatched"] = raw_total + group_total = sum(group_counts.values()) + classified_total = sum(final_counts.values()) + return { + "raw_total": raw_total, + "classified_total": classified_total, + "difference": classified_total - raw_total, + "group_total": classified_total, + "group_difference": classified_total - raw_total, + "source_group_total": group_total, + "source_group_difference": group_total - raw_total, + "counts": final_counts, + "items": [ + {"key": key, "label": status_labels[key], "count": int(final_counts.get(key, 0) or 0)} + for key in WEHAGO_FINAL_STATUS_ORDER + ], + "source": "query_projection" if signature else "raw_only", + } + + +def apply_wehago_final_status_counts_to_metric_sections( + metric_sections: list[dict[str, Any]], + final_status_summary: dict[str, Any] | None, +) -> list[dict[str, Any]]: + if clean((final_status_summary or {}).get("source")) not in {"query_projection", "final_status_projection"}: + return metric_sections + counts = (final_status_summary or {}).get("counts") + if not isinstance(counts, dict): + return metric_sections + for section in metric_sections: + status_key = clean(section.get("key")) + if status_key in WEHAGO_FINAL_STATUS_ORDER: + section["count"] = int(counts.get(status_key, 0) or 0) + return metric_sections + + +def _enqueue_compare_refresh_async( + engine: Any, + start_year: int | None, + end_year: int | None, + years_to_enqueue: Iterable[int], +) -> None: + if start_year is None or end_year is None: + return + years = sorted({int(year) for year in years_to_enqueue if int(year or 0) > 0}) + refresh_key = f"{start_year}:{end_year}:{','.join(str(year) for year in years)}" + with _COMPARE_REFRESH_ENQUEUEING_LOCK: + if refresh_key in _COMPARE_REFRESH_ENQUEUEING: + return + _COMPARE_REFRESH_ENQUEUEING.add(refresh_key) + + def _worker() -> None: + try: + try: + with engine.begin() as conn: + backfill_wehago_final_status_projection_from_query_groups(conn, start_year, end_year) + except Exception as exc: + logger.warning("Skipped async final status projection backfill for %s-%s: %s", start_year, end_year, exc) + try: + enqueue_query_projection_rebuild(engine, start_year, end_year) + except Exception as exc: + logger.warning("Skipped async query projection enqueue for %s-%s: %s", start_year, end_year, exc) + try: + enqueue_metric_count_rebuild(engine, start_year, end_year) + except Exception as exc: + logger.warning("Skipped async metric count enqueue for %s-%s: %s", start_year, end_year, exc) + if years: + try: + enqueue_year_snapshot_rebuild(engine, years) + except Exception as exc: + logger.warning("Skipped async year snapshot enqueue for %s-%s: %s", start_year, end_year, exc) + finally: + with _COMPARE_REFRESH_ENQUEUEING_LOCK: + _COMPARE_REFRESH_ENQUEUEING.discard(refresh_key) + + threading.Thread( + target=_worker, + daemon=True, + name=f"wehago-compare-refresh-enqueue-{start_year}-{end_year}", + ).start() + + def _store_metric_counts_cache( conn: Any, start_year: int | None, @@ -1572,6 +2779,180 @@ def _store_query_group_projection( logger.warning("Skipped query group projection write for %s-%s due to lock: %s", start_year, end_year, exc) +def _chunked(items: list[dict[str, Any]], size: int = 1000) -> Iterable[list[dict[str, Any]]]: + for index in range(0, len(items), max(int(size or 1000), 1)): + yield items[index : index + max(int(size or 1000), 1)] + + +def _store_status_query_group_projection( + conn: Any, + start_year: int | None, + end_year: int | None, + status_key: str, + signature: str, + groups: list[dict[str, Any]], +) -> None: + if start_year is None or end_year is None or not clean(status_key) or not clean(signature): + return + normalized_status = clean(status_key) + conn.execute( + text( + """ + DELETE FROM wehago_compare_query_rows + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": clean(signature), + }, + ) + conn.execute( + text( + """ + DELETE FROM wehago_compare_query_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": clean(signature), + }, + ) + group_payloads: list[dict[str, Any]] = [] + row_payloads: list[dict[str, Any]] = [] + for group_index, group in enumerate(groups): + summary = dict(group.get("summary", {}) or {}) + rows = _sanitize_voucher_group_rows(list(group.get("rows", []) or [])) + search_text = " ".join( + filter( + None, + [ + clean(summary.get("voucher_no")), + clean(summary.get("draft_no")), + clean(summary.get("ledger_accounts")), + clean(summary.get("voucher_accounts")), + clean(summary.get("ledger_vendors")), + clean(summary.get("voucher_vendors")), + clean(summary.get("review_reason")), + " ".join(clean(row.get("ledger_desc")) for row in rows), + " ".join(clean(row.get("voucher_desc")) for row in rows), + ], + ) + ) + group_payloads.append( + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": clean(signature), + "group_index": group_index, + "fiscal_year": int(summary.get("fiscal_year") or 0), + "ledger_date": clean(summary.get("ledger_date")), + "proof_date": clean(summary.get("proof_date")), + "voucher_no": clean(summary.get("voucher_no")), + "draft_no": clean(summary.get("draft_no")), + "ledger_row_count": int(summary.get("ledger_row_count") or 0), + "voucher_row_count": int(summary.get("voucher_row_count") or 0), + "ledger_debit": parse_amount(summary.get("ledger_debit")), + "ledger_credit": parse_amount(summary.get("ledger_credit")), + "voucher_debit": parse_amount(summary.get("voucher_debit")), + "voucher_credit": parse_amount(summary.get("voucher_credit")), + "ledger_accounts": clean(summary.get("ledger_accounts")), + "voucher_accounts": clean(summary.get("voucher_accounts")), + "ledger_vendors": clean(summary.get("ledger_vendors")), + "voucher_vendors": clean(summary.get("voucher_vendors")), + "review_reason": clean(summary.get("review_reason")), + "search_text": search_text, + } + ) + for row_index, row in enumerate(rows): + row_payloads.append( + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": clean(signature), + "group_index": group_index, + "row_index": row_index, + "fiscal_year": int(row.get("fiscal_year") or summary.get("fiscal_year") or 0), + "status_label": clean(row.get("status_label") or summary.get("status_label")), + "ledger_date": clean(row.get("ledger_date") or summary.get("ledger_date")), + "proof_date": clean(row.get("proof_date") or summary.get("proof_date")), + "voucher_no": clean(row.get("voucher_no") or summary.get("voucher_no")), + "draft_no": clean(row.get("draft_no") or summary.get("draft_no")), + "ledger_account_name": clean(row.get("ledger_account_name")), + "voucher_account_name": clean(row.get("voucher_account_name")), + "ledger_vendor": clean(row.get("ledger_vendor")), + "voucher_vendor": clean(row.get("voucher_vendor")), + "ledger_debit": parse_amount(row.get("ledger_debit")), + "ledger_credit": parse_amount(row.get("ledger_credit")), + "voucher_debit": parse_amount(row.get("voucher_debit")), + "voucher_credit": parse_amount(row.get("voucher_credit")), + "ledger_desc": clean(row.get("ledger_desc")), + "voucher_desc": clean(row.get("voucher_desc")), + "review_reason": clean(row.get("review_reason")), + "matched_case": clean(row.get("matched_case")), + "ledger_row_key": clean(row.get("ledger_row_key")), + "voucher_row_key": clean(row.get("voucher_row_key")), + "match_identity_key": clean(row.get("match_identity_key")), + } + ) + group_sql = text( + """ + INSERT INTO wehago_compare_query_groups ( + start_year, end_year, status_key, signature, group_index, + fiscal_year, ledger_date, proof_date, voucher_no, draft_no, + ledger_row_count, voucher_row_count, + ledger_debit, ledger_credit, voucher_debit, voucher_credit, + ledger_accounts, voucher_accounts, ledger_vendors, voucher_vendors, + review_reason, search_text, created_at, updated_at + ) VALUES ( + :start_year, :end_year, :status_key, :signature, :group_index, + :fiscal_year, :ledger_date, :proof_date, :voucher_no, :draft_no, + :ledger_row_count, :voucher_row_count, + :ledger_debit, :ledger_credit, :voucher_debit, :voucher_credit, + :ledger_accounts, :voucher_accounts, :ledger_vendors, :voucher_vendors, + :review_reason, :search_text, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ) + row_sql = text( + """ + INSERT INTO wehago_compare_query_rows ( + start_year, end_year, status_key, signature, group_index, row_index, + fiscal_year, status_label, ledger_date, proof_date, voucher_no, draft_no, + ledger_account_name, voucher_account_name, ledger_vendor, voucher_vendor, + ledger_debit, ledger_credit, voucher_debit, voucher_credit, + ledger_desc, voucher_desc, review_reason, matched_case, + ledger_row_key, voucher_row_key, match_identity_key, created_at, updated_at + ) VALUES ( + :start_year, :end_year, :status_key, :signature, :group_index, :row_index, + :fiscal_year, :status_label, :ledger_date, :proof_date, :voucher_no, :draft_no, + :ledger_account_name, :voucher_account_name, :ledger_vendor, :voucher_vendor, + :ledger_debit, :ledger_credit, :voucher_debit, :voucher_credit, + :ledger_desc, :voucher_desc, :review_reason, :matched_case, + :ledger_row_key, :voucher_row_key, :match_identity_key, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ) + for chunk in _chunked(group_payloads): + conn.execute(group_sql, chunk) + for chunk in _chunked(row_payloads): + conn.execute(row_sql, chunk) + _store_active_status_projection_signature(conn, normalized_status, start_year, end_year, signature, len(groups)) + + def _store_query_row_projection( conn: Any, start_year: int | None, @@ -1680,7 +3061,7 @@ def _load_query_page_projection_cache( or not clean(query_hash) ): return None - signature = _query_projection_signature(conn, start_year, end_year) + signature = _effective_query_projection_signature(conn, start_year, end_year) row = conn.execute( text( """ @@ -1725,6 +3106,7 @@ def _load_latest_query_page_projection_cache_any_signature( or not clean(query_hash) ): return None + signature = _effective_query_projection_signature(conn, start_year, end_year) row = conn.execute( text( """ @@ -1734,8 +3116,7 @@ def _load_latest_query_page_projection_cache_any_signature( AND end_year = :end_year AND status_key = :status_key AND query_hash = :query_hash - AND signature LIKE :signature_like - ORDER BY updated_at DESC, created_at DESC + AND signature = :signature LIMIT 1 """ ), @@ -1744,7 +3125,7 @@ def _load_latest_query_page_projection_cache_any_signature( "end_year": end_year, "status_key": status_key, "query_hash": clean(query_hash), - "signature_like": f"%{QUERY_PROJECTION_VERSION}%", + "signature": signature, }, ).scalar_one_or_none() if not row: @@ -1933,12 +3314,19 @@ def _load_query_group_page( ) -> tuple[list[dict[str, Any]], int, str] | None: if start_year is None or end_year is None or normalized_status not in QUERY_VOUCHER_STATUS_KEYS: return None - projection_scope = _find_best_query_projection_scope( - conn, - "wehago_compare_query_groups", - start_year, - end_year, - normalized_status, + status_signature = "" + if normalized_status == "hanmac_unconnected": + status_signature = _active_status_projection_signature(conn, normalized_status, start_year, end_year) + projection_scope = ( + (int(start_year), int(end_year), status_signature) + if status_signature + else _find_best_query_projection_scope( + conn, + "wehago_compare_query_groups", + start_year, + end_year, + normalized_status, + ) ) if not projection_scope: return None @@ -2105,6 +3493,553 @@ def _load_query_group_page( return groups, total_count, next_cursor +def _final_status_label(status_key: str) -> str: + if status_key == "voucher_matched": + return "Matched" + if status_key == "voucher_recheck": + return "Recheck" + if status_key == "voucher_excepted": + return "Excepted" + return "Unmatched" + + +def _voucher_summary_display_row(summary: dict[str, Any]) -> dict[str, Any]: + return { + "fiscal_year": int(summary.get("fiscal_year") or 0), + "ledger_date": clean(summary.get("ledger_date")), + "voucher_no": clean(summary.get("voucher_no")), + "draft_no": clean(summary.get("draft_no")), + "ledger_account_name": clean(summary.get("ledger_accounts")), + "voucher_account_name": clean(summary.get("voucher_accounts")), + "ledger_vendor": clean(summary.get("ledger_vendors")), + "voucher_vendor": clean(summary.get("voucher_vendors")), + "ledger_debit": parse_amount(summary.get("ledger_debit")), + "ledger_credit": parse_amount(summary.get("ledger_credit")), + "voucher_debit": parse_amount(summary.get("voucher_debit")), + "voucher_credit": parse_amount(summary.get("voucher_credit")), + "ledger_desc": clean(summary.get("review_reason")), + "voucher_desc": "", + "review_reason": clean(summary.get("review_reason")), + } + + +def _load_final_wehago_status_group_page( + conn: Any, + start_year: int | None, + end_year: int | None, + normalized_status: str, + voucher_no: str, + draft_no: str, + wehago_account: str, + erp_account: str, + wehago_amount: str, + erp_amount: str, + wehago_vendor: str, + erp_vendor: str, + desc_keyword: str, + offset: int, + limit: int, + cursor: str = "", +) -> tuple[list[dict[str, Any]], int, str] | None: + if start_year is None or end_year is None or normalized_status not in WEHAGO_FINAL_STATUS_ORDER: + return None + signature = _load_latest_query_projection_signature_for_range(conn, start_year, end_year) + if not signature: + return None + + voucher_filter = clean(voucher_no) + draft_filter = clean(draft_no) + wehago_account_filter = normalize_text(wehago_account) + erp_account_filter = normalize_text(erp_account) + wehago_vendor_filter = normalize_text(wehago_vendor) + erp_vendor_filter = normalize_text(erp_vendor) + desc_filter = normalize_text(desc_keyword) + if ( + voucher_filter + or draft_filter + or wehago_account_filter + or erp_account_filter + or clean(wehago_amount) + or clean(erp_amount) + or wehago_vendor_filter + or erp_vendor_filter + or desc_filter + ): + return None + + fast_params = { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + "status_key": normalized_status, + "limit": int(limit), + "offset": int(offset), + } + fast_source_sql = """ + WITH raw_vouchers AS ( + SELECT + fiscal_year, + compare_voucher_no, + MIN(ledger_date) AS raw_ledger_date, + MIN(voucher_no) AS raw_voucher_no + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(compare_voucher_no, '') <> '' + GROUP BY fiscal_year, compare_voucher_no + ), + final_vouchers AS ( + SELECT + r.fiscal_year, + r.compare_voucher_no, + COALESCE(p.final_status, 'voucher_unmatched') AS final_status, + COALESCE(NULLIF(p.ledger_date, ''), r.raw_ledger_date, '') AS final_ledger_date, + COALESCE(NULLIF(p.voucher_no, ''), r.raw_voucher_no, '') AS final_voucher_no + FROM raw_vouchers r + LEFT JOIN wehago_compare_final_status_projection p + ON p.start_year = :start_year + AND p.end_year = :end_year + AND p.signature = :signature + AND p.fiscal_year = r.fiscal_year + AND p.compare_voucher_no = r.compare_voucher_no + ) + """ + try: + total_count = int( + conn.execute( + text(f"{fast_source_sql} SELECT COUNT(*) FROM final_vouchers WHERE final_status = :status_key"), + fast_params, + ).scalar_one() + or 0 + ) + except OperationalError: + return None + if total_count <= 0: + return [], 0, "" + cursor_year = 0 + cursor_compare_no = "" + cursor_text = clean(cursor) + if cursor_text: + try: + raw_year, raw_compare_no = cursor_text.split(":", 1) + cursor_year = int(raw_year or 0) + cursor_compare_no = raw_compare_no + except Exception: + cursor_text = "" + fast_query_sql = f""" + {fast_source_sql} + SELECT * + FROM final_vouchers + WHERE final_status = :status_key + """ + if cursor_text: + fast_query_sql += """ + AND ( + fiscal_year > :cursor_year + OR (fiscal_year = :cursor_year AND compare_voucher_no > :cursor_compare_no) + ) + """ + fast_params["cursor_year"] = cursor_year + fast_params["cursor_compare_no"] = cursor_compare_no + fast_query_sql += """ + ORDER BY fiscal_year ASC, compare_voucher_no ASC + LIMIT :limit + """ + if not cursor_text: + fast_query_sql += " OFFSET :offset" + final_rows = conn.execute(text(fast_query_sql), fast_params).mappings().all() + if not final_rows: + return [], total_count, "" + + compare_values = [clean(row.get("compare_voucher_no")) for row in final_rows if clean(row.get("compare_voucher_no"))] + group_by_compare: dict[tuple[int, str], dict[str, Any]] = {} + if compare_values: + compare_clauses = [] + group_params: dict[str, Any] = { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + } + for index, compare_no in enumerate(compare_values): + compare_clauses.append( + """ + ( + q.fiscal_year = :fiscal_year_{index} + AND CASE + WHEN COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), '') <> '' + AND COALESCE(NULLIF(q.voucher_no, ''), '') <> '' + THEN ( + CASE + WHEN COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), '') GLOB '[0-9][0-9]-[0-9][0-9]' + THEN printf('%04d%s', q.fiscal_year, REPLACE(COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), ''), '-', '')) + ELSE REPLACE(COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), ''), '-', '') + END + ) + || '-' + || TRIM(CASE WHEN INSTR(q.voucher_no, ',') > 0 THEN SUBSTR(q.voucher_no, 1, INSTR(q.voucher_no, ',') - 1) ELSE q.voucher_no END) + ELSE '' + END = :compare_no_{index} + ) + """.replace("{index}", str(index)) + ) + group_params[f"fiscal_year_{index}"] = int(final_rows[index].get("fiscal_year") or 0) + group_params[f"compare_no_{index}"] = compare_no + query_group_rows = conn.execute( + text( + f""" + SELECT q.* + FROM wehago_compare_query_groups q + WHERE q.start_year = :start_year + AND q.end_year = :end_year + AND q.signature = :signature + AND q.status_key IN ('voucher_matched', 'voucher_unmatched', 'voucher_recheck', 'voucher_excepted') + AND ({' OR '.join(compare_clauses)}) + """ + ), + group_params, + ).mappings().all() + priority = { + "voucher_excepted": 1, + "voucher_matched": 2, + "voucher_recheck": 3, + "voucher_unmatched": 4, + } + for query_row in query_group_rows: + row_date = clean(query_row.get("ledger_date")) or clean(query_row.get("proof_date")) + row_voucher = clean(query_row.get("voucher_no")) + first_voucher = clean(row_voucher.split(",", 1)[0]) if row_voucher else "" + if re.fullmatch(r"\d{2}-\d{2}", row_date): + row_date = f"{int(query_row.get('fiscal_year') or 0):04d}-{row_date}" + compare_no = f"{row_date.replace('-', '')}-{first_voucher}" if row_date and first_voucher else "" + key = (int(query_row.get("fiscal_year") or 0), compare_no) + current = group_by_compare.get(key) + row_status = clean(query_row.get("status_key")) + row_rank = 0 if row_status == normalized_status else priority.get(row_status, 99) + current_status = clean((current or {}).get("status_key")) + current_rank = 0 if current_status == normalized_status else priority.get(current_status, 99) + if current is None or row_rank < current_rank or ( + row_rank == current_rank and int(query_row.get("group_index") or 0) < int(current.get("group_index") or 0) + ): + group_by_compare[key] = dict(query_row) + + selected_group_keys = [ + (clean(row.get("status_key")), int(row.get("group_index") or 0)) + for row in group_by_compare.values() + ] + rows_by_key: dict[tuple[str, int], list[dict[str, Any]]] = {} + if selected_group_keys: + clauses = [] + row_params: dict[str, Any] = { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + } + for index, (status_key, group_index) in enumerate(selected_group_keys): + clauses.append(f"(status_key = :status_key_{index} AND group_index = :group_index_{index})") + row_params[f"status_key_{index}"] = status_key + row_params[f"group_index_{index}"] = group_index + detail_rows = conn.execute( + text( + f""" + SELECT * + FROM wehago_compare_query_rows + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + AND ({' OR '.join(clauses)}) + ORDER BY status_key ASC, group_index ASC, row_index ASC + """ + ), + row_params, + ).mappings().all() + for detail_row in detail_rows: + rows_by_key.setdefault( + (clean(detail_row.get("status_key")), int(detail_row.get("group_index") or 0)), + [], + ).append(dict(detail_row)) + + groups: list[dict[str, Any]] = [] + for final_row in final_rows: + key = (int(final_row.get("fiscal_year") or 0), clean(final_row.get("compare_voucher_no"))) + query_row = group_by_compare.get(key, {}) + source_status = clean(query_row.get("status_key")) + group_index = int(query_row.get("group_index") or -1) + ledger_date = clean(query_row.get("ledger_date")) or clean(final_row.get("final_ledger_date")) + voucher_number = clean(query_row.get("voucher_no")) or clean(final_row.get("final_voucher_no")) + summary = { + "fiscal_year": int(final_row.get("fiscal_year") or 0), + "status_label": _final_status_label(normalized_status), + "ledger_date": ledger_date, + "proof_date": clean(query_row.get("proof_date")), + "voucher_no": voucher_number, + "draft_no": clean(query_row.get("draft_no")), + "ledger_row_count": int(query_row.get("ledger_row_count") or 0), + "voucher_row_count": int(query_row.get("voucher_row_count") or 0), + "ledger_debit": parse_amount(query_row.get("ledger_debit")), + "ledger_credit": parse_amount(query_row.get("ledger_credit")), + "voucher_debit": parse_amount(query_row.get("voucher_debit")), + "voucher_credit": parse_amount(query_row.get("voucher_credit")), + "ledger_accounts": clean(query_row.get("ledger_accounts")), + "voucher_accounts": clean(query_row.get("voucher_accounts")), + "ledger_vendors": clean(query_row.get("ledger_vendors")), + "voucher_vendors": clean(query_row.get("voucher_vendors")), + "review_reason": clean(query_row.get("review_reason")) or ("WEHAGO 원장 미분류 전표" if not query_row else ""), + } + detail_rows = rows_by_key.get((source_status, group_index), []) if query_row else [] + if not detail_rows: + detail_rows = [_voucher_summary_display_row(summary)] + groups.append({"summary": summary, "rows": _sanitize_voucher_group_rows(detail_rows)}) + last_row = final_rows[-1] + next_cursor = ( + f"{int(last_row.get('fiscal_year') or 0)}:{clean(last_row.get('compare_voucher_no'))}" + if len(final_rows) >= int(limit or 0) + else "" + ) + return groups, total_count, next_cursor + + params = { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + "status_key": normalized_status, + "voucher_keyword": voucher_filter, + "voucher_like": f"%{voucher_filter}%", + "draft_keyword": draft_filter, + "draft_like": f"%{draft_filter}%", + "wehago_account_keyword": wehago_account_filter, + "wehago_account_like": f"%{wehago_account_filter}%", + "erp_account_keyword": erp_account_filter, + "erp_account_like": f"%{erp_account_filter}%", + "wehago_vendor_keyword": wehago_vendor_filter, + "wehago_vendor_like": f"%{wehago_vendor_filter}%", + "erp_vendor_keyword": erp_vendor_filter, + "erp_vendor_like": f"%{erp_vendor_filter}%", + "desc_keyword": desc_filter, + "desc_like": f"%{desc_filter}%", + "wehago_amount_keyword": clean(wehago_amount), + "erp_amount_keyword": clean(erp_amount), + "limit": int(limit), + "offset": int(offset), + } + source_sql = """ + WITH raw_vouchers AS ( + SELECT + fiscal_year, + compare_voucher_no, + MIN(ledger_date) AS raw_ledger_date, + MIN(voucher_no) AS raw_voucher_no + FROM wehago_ledger_rows + WHERE fiscal_year BETWEEN :start_year AND :end_year + AND COALESCE(compare_voucher_no, '') <> '' + GROUP BY fiscal_year, compare_voucher_no + ), + final_vouchers AS ( + SELECT + r.fiscal_year, + r.compare_voucher_no, + COALESCE(p.final_status, 'voucher_unmatched') AS final_status, + COALESCE(NULLIF(p.ledger_date, ''), r.raw_ledger_date, '') AS final_ledger_date, + COALESCE(NULLIF(p.voucher_no, ''), r.raw_voucher_no, '') AS final_voucher_no + FROM raw_vouchers r + LEFT JOIN wehago_compare_final_status_projection p + ON p.start_year = :start_year + AND p.end_year = :end_year + AND p.signature = :signature + AND p.fiscal_year = r.fiscal_year + AND p.compare_voucher_no = r.compare_voucher_no + ), + ranked_groups AS ( + SELECT + q.*, + ROW_NUMBER() OVER ( + PARTITION BY q.fiscal_year, + CASE + WHEN COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), '') <> '' + AND COALESCE(NULLIF(q.voucher_no, ''), '') <> '' + THEN REPLACE(COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), ''), '-', '') + || '-' + || TRIM(CASE WHEN INSTR(q.voucher_no, ',') > 0 THEN SUBSTR(q.voucher_no, 1, INSTR(q.voucher_no, ',') - 1) ELSE q.voucher_no END) + ELSE '' + END + ORDER BY + CASE WHEN q.status_key = :status_key THEN 0 ELSE 1 END, + CASE q.status_key + WHEN 'voucher_excepted' THEN 1 + WHEN 'voucher_matched' THEN 2 + WHEN 'voucher_recheck' THEN 3 + WHEN 'voucher_unmatched' THEN 4 + ELSE 99 + END, + q.group_index + ) AS row_rank, + CASE + WHEN COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), '') <> '' + AND COALESCE(NULLIF(q.voucher_no, ''), '') <> '' + THEN REPLACE(COALESCE(NULLIF(q.ledger_date, ''), NULLIF(q.proof_date, ''), ''), '-', '') + || '-' + || TRIM(CASE WHEN INSTR(q.voucher_no, ',') > 0 THEN SUBSTR(q.voucher_no, 1, INSTR(q.voucher_no, ',') - 1) ELSE q.voucher_no END) + ELSE '' + END AS q_compare_voucher_no + FROM wehago_compare_query_groups q + WHERE q.start_year = :start_year + AND q.end_year = :end_year + AND q.signature = :signature + AND q.fiscal_year BETWEEN :start_year AND :end_year + AND q.status_key IN ('voucher_matched', 'voucher_unmatched', 'voucher_recheck', 'voucher_excepted') + ), + final_rows AS ( + SELECT + f.fiscal_year AS final_fiscal_year, + f.compare_voucher_no AS final_compare_voucher_no, + f.final_status, + f.final_ledger_date, + f.final_voucher_no, + q.* + FROM final_vouchers f + LEFT JOIN ranked_groups q + ON q.fiscal_year = f.fiscal_year + AND q.q_compare_voucher_no = f.compare_voucher_no + AND q.row_rank = 1 + WHERE f.final_status = :status_key + ) + """ + filter_sql = """ + (:voucher_keyword = '' OR COALESCE(voucher_no, final_voucher_no, '') LIKE :voucher_like) + AND (:draft_keyword = '' OR COALESCE(draft_no, '') LIKE :draft_like) + AND (:wehago_account_keyword = '' OR COALESCE(ledger_accounts, '') LIKE :wehago_account_like) + AND (:erp_account_keyword = '' OR COALESCE(voucher_accounts, '') LIKE :erp_account_like) + AND (:wehago_vendor_keyword = '' OR COALESCE(ledger_vendors, '') LIKE :wehago_vendor_like) + AND (:erp_vendor_keyword = '' OR COALESCE(voucher_vendors, '') LIKE :erp_vendor_like) + AND (:desc_keyword = '' OR COALESCE(search_text, '') LIKE :desc_like) + AND ( + :wehago_amount_keyword = '' + OR CAST(ABS(COALESCE(ledger_debit, 0)) AS TEXT) LIKE '%' || :wehago_amount_keyword || '%' + OR CAST(ABS(COALESCE(ledger_credit, 0)) AS TEXT) LIKE '%' || :wehago_amount_keyword || '%' + ) + AND ( + :erp_amount_keyword = '' + OR CAST(ABS(COALESCE(voucher_debit, 0)) AS TEXT) LIKE '%' || :erp_amount_keyword || '%' + OR CAST(ABS(COALESCE(voucher_credit, 0)) AS TEXT) LIKE '%' || :erp_amount_keyword || '%' + ) + """ + try: + total_count = int( + conn.execute( + text(f"{source_sql} SELECT COUNT(*) FROM final_rows WHERE {filter_sql}"), + params, + ).scalar_one() + or 0 + ) + except OperationalError: + return None + if total_count <= 0: + return [], 0, "" + + cursor_year = 0 + cursor_compare_no = "" + cursor_text = clean(cursor) + if cursor_text: + try: + raw_year, raw_compare_no = cursor_text.split(":", 1) + cursor_year = int(raw_year or 0) + cursor_compare_no = raw_compare_no + except Exception: + cursor_text = "" + query_sql = f""" + {source_sql} + SELECT * + FROM final_rows + WHERE {filter_sql} + """ + if cursor_text: + query_sql += """ + AND ( + final_fiscal_year > :cursor_year + OR (final_fiscal_year = :cursor_year AND final_compare_voucher_no > :cursor_compare_no) + ) + """ + params["cursor_year"] = cursor_year + params["cursor_compare_no"] = cursor_compare_no + query_sql += """ + ORDER BY final_fiscal_year ASC, final_compare_voucher_no ASC + LIMIT :limit + """ + if not cursor_text: + query_sql += " OFFSET :offset" + rows = conn.execute(text(query_sql), params).mappings().all() + if not rows: + return [], total_count, "" + + selected_group_keys = [ + (int(row.get("group_index") or -1), clean(row.get("status_key"))) + for row in rows + if row.get("group_index") is not None and clean(row.get("status_key")) + ] + rows_by_key: dict[tuple[str, int], list[dict[str, Any]]] = {} + if selected_group_keys: + clauses = [] + row_params = { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + } + for index, (group_index, status_key) in enumerate(selected_group_keys): + clauses.append(f"(status_key = :status_key_{index} AND group_index = :group_index_{index})") + row_params[f"status_key_{index}"] = status_key + row_params[f"group_index_{index}"] = group_index + detail_rows = conn.execute( + text( + f""" + SELECT * + FROM wehago_compare_query_rows + WHERE start_year = :start_year + AND end_year = :end_year + AND signature = :signature + AND ({' OR '.join(clauses)}) + ORDER BY status_key ASC, group_index ASC, row_index ASC + """ + ), + row_params, + ).mappings().all() + for detail_row in detail_rows: + key = (clean(detail_row.get("status_key")), int(detail_row.get("group_index") or 0)) + rows_by_key.setdefault(key, []).append(dict(detail_row)) + + groups: list[dict[str, Any]] = [] + for row in rows: + source_status = clean(row.get("status_key")) or normalized_status + group_index = int(row.get("group_index") or -1) + ledger_date = clean(row.get("ledger_date")) or clean(row.get("final_ledger_date")) + voucher_number = clean(row.get("voucher_no")) or clean(row.get("final_voucher_no")) + summary = { + "fiscal_year": int(row.get("final_fiscal_year") or row.get("fiscal_year") or 0), + "status_label": _final_status_label(normalized_status), + "ledger_date": ledger_date, + "proof_date": clean(row.get("proof_date")), + "voucher_no": voucher_number, + "draft_no": clean(row.get("draft_no")), + "ledger_row_count": int(row.get("ledger_row_count") or 0), + "voucher_row_count": int(row.get("voucher_row_count") or 0), + "ledger_debit": parse_amount(row.get("ledger_debit")), + "ledger_credit": parse_amount(row.get("ledger_credit")), + "voucher_debit": parse_amount(row.get("voucher_debit")), + "voucher_credit": parse_amount(row.get("voucher_credit")), + "ledger_accounts": clean(row.get("ledger_accounts")), + "voucher_accounts": clean(row.get("voucher_accounts")), + "ledger_vendors": clean(row.get("ledger_vendors")), + "voucher_vendors": clean(row.get("voucher_vendors")), + "review_reason": clean(row.get("review_reason")) or ("WEHAGO 원장 미분류 전표" if group_index < 0 else ""), + } + detail_rows = rows_by_key.get((source_status, group_index), []) if group_index >= 0 else [] + groups.append({"summary": summary, "rows": _sanitize_voucher_group_rows(detail_rows)}) + last_row = rows[-1] + next_cursor = ( + f"{int(last_row.get('final_fiscal_year') or 0)}:{clean(last_row.get('final_compare_voucher_no'))}" + if len(rows) >= int(limit or 0) + else "" + ) + return groups, total_count, next_cursor + + def _load_query_row_page( conn: Any, start_year: int | None, @@ -2464,6 +4399,33 @@ def _merge_export_cache_voucher_counts( return merged +def _apply_hanmac_unconnected_source_count( + conn: Any, + counts: dict[str, int], + start_year: int | None, + end_year: int | None, +) -> dict[str, int]: + merged = {status_key: int(counts.get(status_key, 0) or 0) for status_key, _, _ in STATUS_META} + if start_year is None or end_year is None: + return merged + projected_counts = _load_active_status_projection_counts_for_range( + conn, + start_year, + end_year, + {"hanmac_unconnected", *COMPACT_EXPORT_STATUS_KEYS}, + ) + if "hanmac_unconnected" in projected_counts: + merged["hanmac_unconnected"] = int(projected_counts.get("hanmac_unconnected") or 0) + else: + merged["hanmac_unconnected"] = len( + _load_hanmac_unconnected_source_groups(conn, int(start_year), int(end_year)) + ) + for status_key in sorted(COMPACT_EXPORT_STATUS_KEYS): + if status_key in projected_counts: + merged[status_key] = int(projected_counts.get(status_key) or 0) + return merged + + def _count_distinct_group_summary_tokens( groups: Iterable[dict[str, Any]], summary_key: str, @@ -2608,9 +4570,10 @@ def _resolved_metric_counts(conn: Any, start_year: int | None, end_year: int | N def get_dashboard_metric_counts(conn: Any, start_year: int | None, end_year: int | None) -> dict[str, int]: cached = _load_metric_counts_cache(conn, start_year, end_year) if cached is not None: - return cached + return _apply_hanmac_unconnected_source_count(conn, cached, start_year, end_year) counts = _resolved_metric_counts(conn, start_year, end_year) counts["bridge_expense_review"] = _count_bridge_expense_review_candidates(conn, start_year, end_year) + counts = _apply_hanmac_unconnected_source_count(conn, counts, start_year, end_year) _store_metric_counts_cache(conn, start_year, end_year, counts) return counts @@ -2635,6 +4598,7 @@ def _rebuild_metric_count_caches( counts = _resolved_metric_counts(conn, start_year, end_year) counts["bridge_expense_review"] = _count_bridge_expense_review_candidates(conn, start_year, end_year) counts = _apply_cached_adjacent_year_demotion_count_adjustment(conn, counts, start_year, end_year) + counts = _apply_hanmac_unconnected_source_count(conn, counts, start_year, end_year) snapshot_state = _get_compare_snapshot_state(conn, start_year, end_year) _store_metric_counts_cache(conn, start_year, end_year, counts) _store_summary_range_cache(conn, start_year, end_year, counts, snapshot_state) @@ -2671,6 +4635,8 @@ def _rebuild_compare_query_projection( sqlite_conn = sqlite3.connect(db_path) sqlite_conn.row_factory = sqlite3.Row try: + for year in range(int(start_year), int(end_year) + 1): + _ensure_year_export_row_cache(conn, year) export_counts = project_range(sqlite_conn, int(start_year), int(end_year)) finally: sqlite_conn.close() @@ -2710,6 +4676,7 @@ def _rebuild_compare_query_projection( ) _store_query_group_projection(conn, start_year, end_year, voucher_sections) _store_query_row_projection(conn, start_year, end_year, rows_by_status) + _store_wehago_final_status_projection(conn, start_year, end_year, voucher_sections) _prewarm_query_page_projection_cache(conn, start_year, end_year, rows_by_status, voucher_sections) return counts, snapshot_state @@ -2832,6 +4799,7 @@ def get_dashboard_metric_counts_nonblocking( init_wehago_compare_db(engine) pending = False years_to_enqueue: list[int] = [] + compact_status_ready = False with engine.begin() as conn: snapshot_state = _get_compare_snapshot_state(conn, start_year, end_year) pending = bool( @@ -2841,211 +4809,63 @@ def get_dashboard_metric_counts_nonblocking( or (snapshot_state or {}).get("running") ) - fast_projection_counts = _resolve_fast_projection_counts_from_cached_rows( - engine, - conn, - start_year, - end_year, - ) query_projection = _load_query_metric_projection(conn, start_year, end_year) - if fast_projection_counts is not None and ( - pending - or not query_projection - or not isinstance(query_projection.get("counts"), dict) - ): - try: - _store_metric_counts_cache(conn, start_year, end_year, fast_projection_counts) - _store_summary_range_cache( - conn, - start_year, - end_year, - fast_projection_counts, - snapshot_state or {"ready": [], "stale": [], "missing": []}, - ) - except OperationalError: - pass - return _merge_export_cache_voucher_counts(conn, fast_projection_counts, start_year, end_year), pending - if query_projection and isinstance(query_projection.get("counts"), dict): - query_counts = query_projection["counts"] - if fast_projection_counts is not None: - voucher_like_keys = ( - "voucher_matched", - "erp_voucher_matched", - "voucher_unmatched", - "erp_voucher_unmatched", - "voucher_recheck", - "hanmac_unconnected", - ) - projection_total = sum(int(query_counts.get(key, 0) or 0) for key in voucher_like_keys) - fast_total = sum(int(fast_projection_counts.get(key, 0) or 0) for key in voucher_like_keys) - # Prefer the reconstructed counts when the stored projection is clearly lagging behind - # the cached status rows. This keeps the dashboard from getting stuck on stale low values - # after matching rules change or a projection rebuild is still queued. - if fast_total > 0 and ( - projection_total <= 0 - or projection_total * 2 < fast_total - ): - try: - _store_metric_counts_cache(conn, start_year, end_year, fast_projection_counts) - _store_summary_range_cache( - conn, - start_year, - end_year, - fast_projection_counts, - snapshot_state or {"ready": [], "stale": [], "missing": []}, - ) - except OperationalError: - pass - return _merge_export_cache_voucher_counts(conn, fast_projection_counts, start_year, end_year), pending - return _merge_export_cache_voucher_counts(conn, query_counts, start_year, end_year), pending - - if fast_projection_counts is not None: - try: - _store_metric_counts_cache(conn, start_year, end_year, fast_projection_counts) - _store_summary_range_cache( - conn, - start_year, - end_year, - fast_projection_counts, - snapshot_state or {"ready": [], "stale": [], "missing": []}, - ) - except OperationalError: - pass - return _merge_export_cache_voucher_counts(conn, fast_projection_counts, start_year, end_year), pending + query_counts = { + status_key: int(query_projection["counts"].get(status_key, 0) or 0) + for status_key, _, _ in STATUS_META + } + query_counts = _apply_hanmac_unconnected_source_count(conn, query_counts, start_year, end_year) + compact_status_ready = _all_voucher_status_projections_ready(conn, start_year, end_year) + if compact_status_ready: + pending = False + if _active_query_projection_signature(conn, start_year, end_year): + return query_counts, False + return query_counts, pending cached_summary = _load_summary_range_cache(conn, start_year, end_year) if cached_summary: counts = cached_summary.get("counts") if isinstance(counts, dict): - return _merge_export_cache_voucher_counts(conn, counts, start_year, end_year), False + coerced = _coerce_metric_counts_payload(counts) or _empty_metric_counts() + coerced = _apply_hanmac_unconnected_source_count(conn, coerced, start_year, end_year) + return coerced, False cached = _load_metric_counts_cache(conn, start_year, end_year) if cached is not None: - return _merge_export_cache_voucher_counts(conn, cached, start_year, end_year), False + return _apply_hanmac_unconnected_source_count(conn, cached, start_year, end_year), False fallback_summary = _load_latest_summary_range_cache_any_signature(conn, start_year, end_year) - fallback_counts = None + fallback = None if fallback_summary and isinstance(fallback_summary.get("counts"), dict): - fallback_counts = { - status_key: int(fallback_summary["counts"].get(status_key, 0) or 0) - for status_key, _, _ in STATUS_META - } - fallback = fallback_counts or _load_latest_metric_counts_cache_any_signature(conn, start_year, end_year) - - ready_years = [ - int(year) - for year in ((snapshot_state or {}).get("ready") or []) - if start_year is None or end_year is None or int(start_year) <= int(year) <= int(end_year) - ] - if pending and ready_years: - ready_only_counts = _restore_metric_counts_from_cached_year_payloads( - conn, - start_year, - end_year, - lightweight=True, - selected_years=ready_years, - ) - if ready_only_counts is not None and any(int(ready_only_counts.get(status_key, 0) or 0) for status_key, _, _ in STATUS_META): - ready_only_counts = _merge_export_cache_voucher_counts(conn, ready_only_counts, min(ready_years), max(ready_years)) - fallback = ready_only_counts - - if fallback is None and start_year is not None and end_year is not None: - fallback = _restore_metric_counts_from_cached_year_payloads( - conn, - start_year, - end_year, - lightweight=True, - ) - if fallback is not None: - fallback.setdefault("bridge_expense_review", 0) - try: - _store_metric_counts_cache(conn, start_year, end_year, fallback) - _store_summary_range_cache( - conn, - start_year, - end_year, - fallback, - snapshot_state or {"ready": [], "stale": [], "missing": []}, - ) - except OperationalError: - pass - - if ( - (fallback is None or not any(int(fallback.get(status_key, 0) or 0) for status_key, _, _ in STATUS_META)) - and start_year is not None - and end_year is not None - ): - direct_counts = _empty_metric_counts() - direct_counts.update(_fallback_metric_counts_from_db(conn, start_year, end_year)) - direct_counts.update(_voucher_metric_counts_from_db(conn, start_year, end_year)) - direct_counts["bridge_expense_review"] = _count_bridge_expense_review_candidates(conn, start_year, end_year) - direct_counts = _apply_cached_adjacent_year_demotion_count_adjustment(conn, direct_counts, start_year, end_year) - fallback = direct_counts - try: - _store_metric_counts_cache(conn, start_year, end_year, direct_counts) - _store_summary_range_cache( - conn, - start_year, - end_year, - direct_counts, - snapshot_state or {"ready": [], "stale": [], "missing": []}, - ) - except OperationalError: - pass - - # If older fallback caches claim ERP voucher-unmatched is zero while raw voucher-only - # rows clearly exist, refresh only the voucher-group counts from cached status rows so - # the dashboard card and grouped detail table stay consistent without a heavy rebuild. - if ( - fallback - and int(fallback.get("voucher_only", 0) or 0) > 0 - and int(fallback.get("erp_voucher_unmatched", 0) or 0) == 0 - and int(fallback.get("hanmac_unconnected", 0) or 0) == 0 - and start_year is not None - and end_year is not None - ): - rows_by_status = _get_cached_status_rows_by_range(engine, start_year, end_year) - voucher_sections = _get_cached_voucher_sections_by_range(engine, start_year, end_year, rows_by_status) - voucher_sections = _apply_bridge_expense_promotions_to_voucher_sections( - conn, - start_year, - end_year, - voucher_sections, - ) - refreshed = dict(fallback) - refreshed["voucher_matched"] = len(voucher_sections.get("voucher_matched", [])) - refreshed["erp_voucher_matched"] = len(voucher_sections.get("erp_voucher_matched", [])) - refreshed["voucher_unmatched"] = len(voucher_sections.get("voucher_unmatched", [])) - refreshed["erp_voucher_unmatched"] = len(voucher_sections.get("erp_voucher_unmatched", [])) - refreshed["voucher_recheck"] = len(voucher_sections.get("voucher_recheck", [])) - refreshed["voucher_excepted"] = len(voucher_sections.get("voucher_excepted", [])) - refreshed["hanmac_unconnected"] = len(voucher_sections.get("hanmac_unconnected", [])) - refreshed = _merge_bridge_expense_count(conn, refreshed, start_year, end_year) - _store_metric_counts_cache(conn, start_year, end_year, refreshed) - _store_summary_range_cache(conn, start_year, end_year, refreshed, snapshot_state or {"ready": [], "stale": [], "missing": []}) - return refreshed, pending - + fallback = _coerce_metric_counts_payload(fallback_summary.get("counts")) + if fallback is None: + fallback = _load_latest_query_metric_projection_any_signature(conn, start_year, end_year) + if fallback is None: + fallback = _load_latest_metric_counts_cache_any_signature_loose(conn, start_year, end_year) if fallback is None: fallback = _empty_metric_counts() - fallback = _merge_export_cache_voucher_counts(conn, fallback, start_year, end_year) + fallback = _apply_hanmac_unconnected_source_count(conn, fallback, start_year, end_year) + compact_status_ready = _all_voucher_status_projections_ready(conn, start_year, end_year) + if compact_status_ready: + pending = False + years_to_enqueue = [ int(year) for year in ((snapshot_state or {}).get("missing") or []) + ((snapshot_state or {}).get("stale") or []) ] - if not years_to_enqueue and start_year is not None and end_year is not None and not pending: + if not compact_status_ready and not years_to_enqueue and start_year is not None and end_year is not None and not pending: years_to_enqueue = list(range(int(start_year), int(end_year) + 1)) pending = True - try: - enqueue_query_projection_rebuild(engine, start_year, end_year) - enqueue_metric_count_rebuild(engine, start_year, end_year) - has_fallback_data = any(int((fallback or {}).get(status_key, 0) or 0) for status_key, _, _ in STATUS_META) - if years_to_enqueue and not has_fallback_data: - enqueue_year_snapshot_rebuild(engine, years_to_enqueue) - except Exception: - pending = True + has_fallback_data = any(int((fallback or {}).get(status_key, 0) or 0) for status_key, _, _ in STATUS_META) + _enqueue_compare_refresh_async( + engine, + start_year, + end_year, + years_to_enqueue if not has_fallback_data else [], + ) return fallback or _empty_metric_counts(), pending @@ -3842,6 +5662,64 @@ def init_wehago_compare_db(engine: Any) -> None: """ ) ) + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS wehago_status_projection_groups ( + start_year INTEGER NOT NULL, + end_year INTEGER NOT NULL, + status_key TEXT NOT NULL, + signature TEXT NOT NULL, + group_index INTEGER NOT NULL, + fiscal_year INTEGER NOT NULL DEFAULT 0, + ledger_date TEXT NOT NULL DEFAULT '', + proof_date TEXT NOT NULL DEFAULT '', + voucher_no TEXT NOT NULL DEFAULT '', + draft_no TEXT NOT NULL DEFAULT '', + ledger_row_count INTEGER NOT NULL DEFAULT 0, + voucher_row_count INTEGER NOT NULL DEFAULT 0, + ledger_debit REAL NOT NULL DEFAULT 0, + ledger_credit REAL NOT NULL DEFAULT 0, + voucher_debit REAL NOT NULL DEFAULT 0, + voucher_credit REAL NOT NULL DEFAULT 0, + ledger_accounts TEXT NOT NULL DEFAULT '', + voucher_accounts TEXT NOT NULL DEFAULT '', + ledger_vendors TEXT NOT NULL DEFAULT '', + voucher_vendors TEXT NOT NULL DEFAULT '', + review_reason TEXT NOT NULL DEFAULT '', + search_text TEXT NOT NULL DEFAULT '', + summary_json TEXT NOT NULL DEFAULT '{}', + rows_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (start_year, end_year, status_key, signature, group_index) + ) + """ + ) + ) + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS wehago_compare_final_status_projection ( + start_year INTEGER NOT NULL, + end_year INTEGER NOT NULL, + signature TEXT NOT NULL, + identity_key TEXT NOT NULL, + compare_voucher_no TEXT NOT NULL DEFAULT '', + fiscal_year INTEGER NOT NULL DEFAULT 0, + ledger_date TEXT NOT NULL DEFAULT '', + voucher_no TEXT NOT NULL DEFAULT '', + final_status TEXT NOT NULL DEFAULT '', + final_rank INTEGER NOT NULL DEFAULT 99, + source_statuses TEXT NOT NULL DEFAULT '', + source_group_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (start_year, end_year, signature, identity_key) + ) + """ + ) + ) conn.execute( text( """ @@ -3916,6 +5794,10 @@ def init_wehago_compare_db(engine: Any) -> None: conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_compare_query_rows_group_fetch ON wehago_compare_query_rows(start_year, end_year, status_key, signature, group_index, row_index, fiscal_year)")) conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_compare_query_rows_range_signature ON wehago_compare_query_rows(start_year, end_year, signature, fiscal_year, status_key)")) conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_compare_query_page_cache_lookup ON wehago_compare_query_page_cache(status_key, start_year, end_year, signature, updated_at)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_status_projection_lookup ON wehago_status_projection_groups(status_key, start_year, end_year, signature, fiscal_year, group_index)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_status_projection_filter ON wehago_status_projection_groups(status_key, start_year, end_year, signature, voucher_no, draft_no)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_compare_final_projection_status ON wehago_compare_final_status_projection(start_year, end_year, signature, final_status, fiscal_year)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_compare_final_projection_compare_no ON wehago_compare_final_status_projection(start_year, end_year, signature, compare_voucher_no)")) conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_source ON wehago_raw_erp_trace_candidate_cache(fiscal_year, source_mode, source_signature, logic_version, status_key, group_index)")) conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_voucher ON wehago_raw_erp_trace_candidate_cache(fiscal_year, ledger_date, voucher_no)")) conn.execute(text("CREATE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_score ON wehago_raw_erp_trace_candidate_cache(fiscal_year, score DESC)")) @@ -3945,6 +5827,7 @@ def init_wehago_compare_db(engine: Any) -> None: except OperationalError: pass _try_normalize_compare_background_jobs(conn) + _prune_obsolete_compare_logic_caches(conn) _WEHAGO_COMPARE_DB_READY = True @@ -5163,6 +7046,7 @@ def build_account_options_from_rows(rows_by_status: dict[str, list[dict[str, Any def build_account_options_from_db(conn: Any, start_year: int | None, end_year: int | None, limit: int = 3000) -> dict[str, list[str]]: + erp_start_year, erp_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) wehago_rows = conn.execute( text( f""" @@ -5174,7 +7058,7 @@ def build_account_options_from_db(conn: Any, start_year: int | None, end_year: i LIMIT :limit """ ), - {"start_year": start_year, "end_year": end_year, "limit": limit}, + {"start_year": erp_start_year or start_year, "end_year": erp_end_year or end_year, "limit": limit}, ).fetchall() erp_rows = conn.execute( text( @@ -5359,11 +7243,35 @@ def _get_or_create_year_cached_sections(conn: Any, year: int) -> dict[str, dict[ "parse_signature": parse_signature, }, ).first() + if not cached: + cached = conn.execute( + text( + """ + SELECT payload_json + FROM wehago_result_row_cache + WHERE fiscal_year = :fiscal_year + AND ledger_result_path = :ledger_result_path + AND ledger_result_mtime = :ledger_result_mtime + AND voucher_result_path = :voucher_result_path + AND voucher_result_mtime = :voucher_result_mtime + AND COALESCE(payload_json, '') <> '' + ORDER BY created_at DESC + LIMIT 1 + """ + ), + { + "fiscal_year": year, + "ledger_result_path": ledger_path, + "ledger_result_mtime": ledger_mtime, + "voucher_result_path": voucher_path, + "voucher_result_mtime": voucher_mtime, + }, + ).first() if cached and cached[0]: try: payload = json.loads(str(cached[0])) if isinstance(payload, dict): - return _append_substitution_review_rows(payload) + return payload except Exception: pass @@ -5751,8 +7659,10 @@ def _refresh_year_resolved_sections(conn: Any, year: int) -> dict[str, dict[str, "amount_mismatch": list((sections or {}).get("amount_mismatch", {}).get("rows", [])), "voucher_only": list((sections or {}).get("voucher_only", {}).get("rows", [])), } - voucher_sections = _build_voucher_sections_with_bridge_promotions(conn, year, year, rows_by_status) - _store_year_export_row_cache(conn, year, signature, voucher_sections) + voucher_sections: dict[str, list[dict[str, Any]]] = {} + if ENABLE_YEAR_SNAPSHOT_EXPORT_ROW_CACHE: + voucher_sections = _build_voucher_sections_with_bridge_promotions(conn, year, year, rows_by_status) + _store_year_export_row_cache(conn, year, signature, voucher_sections) _upsert_snapshot_status( conn, year, @@ -5765,6 +7675,7 @@ def _refresh_year_resolved_sections(conn: Any, year: int) -> dict[str, dict[str, def _build_db_state_signature(conn: Any, start_year: int | None, end_year: int | None) -> str: + erp_start_year, erp_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) source = conn.execute( text( """ @@ -5776,6 +7687,17 @@ def _build_db_state_signature(conn: Any, start_year: int | None, end_year: int | ), {"start_year": start_year, "end_year": end_year}, ).first() + erp_source = conn.execute( + text( + """ + SELECT COALESCE(MAX(imported_at), ''), COUNT(*) + FROM wehago_source_files + WHERE (:start_year IS NULL OR year_hint >= :start_year) + AND (:end_year IS NULL OR year_hint <= :end_year) + """ + ), + {"start_year": erp_start_year, "end_year": erp_end_year}, + ).first() comparison = conn.execute( text( """ @@ -5830,7 +7752,7 @@ def _build_db_state_signature(conn: Any, start_year: int | None, end_year: int | bridge_settings_sig = _bridge_review_settings_signature(bridge_settings) return ( f"{_current_logic_signature()}|" - f"s:{source[0]}:{source[1]}|c:{comparison[0]}|" + f"s:{source[0]}:{source[1]}|es:{erp_source[0]}:{erp_source[1]}|c:{comparison[0]}|" f"r:{recheck[0]}:{recheck[1]}|p:{pair[0]}:{pair[1]}|" f"b:{bridge_settings_sig}:{clean((bridge_settings_row or {}).get('updated_at'))}" ) @@ -5946,10 +7868,10 @@ def _normalize_compare_background_jobs(conn: Any) -> None: """ ) ).mappings().all() - now = datetime.now() + now = datetime.utcnow() queued_keep: dict[tuple[Any, ...], str] = {} queued_drop: list[str] = [] - stale_running: list[str] = [] + stale_running: list[dict[str, Any]] = [] for row in rows: job_key = clean(row.get("job_key")) job_type = clean(row.get("job_type")) @@ -5964,15 +7886,50 @@ def _normalize_compare_background_jobs(conn: Any) -> None: if state == "running": started_at = _parse_db_timestamp(row.get("started_at")) or _parse_db_timestamp(row.get("updated_at")) or _parse_db_timestamp(row.get("created_at")) if started_at and (now - started_at).total_seconds() >= COMPARE_JOB_STALE_RUNNING_SEC: - stale_running.append(job_key) + stale_running.append( + { + "job_key": job_key, + "job_type": job_type, + "payload": payload, + } + ) continue if state != "queued" or scope is None: continue + if job_type == "year_snapshot_rebuild": + year = int(payload.get("fiscal_year") or 0) + signature = clean(payload.get("signature")) + status_row = _load_snapshot_status_map(conn, [year]).get(year) if year > 0 else {} + stored_signature = clean((status_row or {}).get("snapshot_signature")) + if ( + year > 0 + and signature + and clean((status_row or {}).get("state")).lower() == "ready" + and stored_signature + and ( + stored_signature == signature + or _snapshot_signature_equivalent(stored_signature, signature) + ) + ): + queued_drop.append(job_key) + continue if scope in queued_keep: queued_drop.append(job_key) continue queued_keep[scope] = job_key - for job_key in stale_running: + for job in stale_running: + job_key = clean(job.get("job_key")) + payload = job.get("payload") if isinstance(job.get("payload"), dict) else {} + if clean(job.get("job_type")) == "year_snapshot_rebuild": + year = int((payload or {}).get("fiscal_year") or 0) + if year > 0: + _upsert_snapshot_status( + conn, + year, + signature=clean((payload or {}).get("signature")), + state="failed", + error_message="abandoned stale running job cleaned during startup", + ) conn.execute( text( """ @@ -5991,6 +7948,51 @@ def _normalize_compare_background_jobs(conn: Any) -> None: text("DELETE FROM wehago_background_jobs WHERE job_key = :job_key"), {"job_key": job_key}, ) + running_status_rows = conn.execute( + text( + """ + SELECT fiscal_year, snapshot_signature + FROM wehago_snapshot_status + WHERE state = 'running' + """ + ) + ).mappings().all() + if not running_status_rows: + return + active_jobs = conn.execute( + text( + """ + SELECT payload_json + FROM wehago_background_jobs + WHERE job_type = 'year_snapshot_rebuild' + AND state IN ('queued', 'running') + """ + ) + ).mappings().all() + active_snapshot_keys: set[tuple[int, str]] = set() + for row in active_jobs: + try: + payload = json.loads(clean(row.get("payload_json")) or "{}") + except Exception: + payload = {} + if not isinstance(payload, dict): + continue + year = int(payload.get("fiscal_year") or 0) + signature = clean(payload.get("signature")) + if year > 0 and signature: + active_snapshot_keys.add((year, signature)) + for row in running_status_rows: + year = int(row.get("fiscal_year") or 0) + signature = clean(row.get("snapshot_signature")) + if year <= 0 or (year, signature) in active_snapshot_keys: + continue + _upsert_snapshot_status( + conn, + year, + signature=signature, + state="failed", + error_message="orphan running snapshot status reconciled without active job", + ) def _normalize_compare_background_jobs_once(conn: Any, *, force: bool = False) -> None: @@ -6017,6 +8019,67 @@ def _try_normalize_compare_background_jobs(conn: Any, *, force: bool = False) -> return +def _prune_obsolete_compare_logic_caches(conn: Any) -> None: + try: + query_like = f"{QUERY_PROJECTION_VERSION}|%" + status_like = f"{STATUS_PROJECTION_VERSION}%" + metric_like = f"{METRIC_COUNT_CACHE_VERSION}|%" + for table_name in ( + "wehago_compare_query_metrics", + "wehago_compare_query_groups", + "wehago_compare_query_rows", + "wehago_compare_query_page_cache", + ): + conn.execute( + text( + f""" + DELETE FROM {table_name} + WHERE signature NOT LIKE :query_signature_like + AND signature NOT LIKE :status_signature_like + """ + ), + { + "query_signature_like": query_like, + "status_signature_like": status_like, + }, + ) + for table_name in ("wehago_metric_count_cache", "wehago_summary_range_cache"): + conn.execute( + text(f"DELETE FROM {table_name} WHERE signature NOT LIKE :signature_like"), + {"signature_like": metric_like}, + ) + conn.execute( + text( + """ + DELETE FROM wehago_status_projection_groups + WHERE signature NOT LIKE :status_signature_like + """ + ), + {"status_signature_like": status_like}, + ) + conn.execute( + text( + """ + DELETE FROM wehago_compare_settings + WHERE ( + setting_key LIKE 'wehago_active_query_projection:%' + OR setting_key LIKE 'wehago_active_status_projection:%' + ) + AND COALESCE(json_extract(setting_json, '$.signature'), '') NOT LIKE :query_signature_like + AND COALESCE(json_extract(setting_json, '$.signature'), '') NOT LIKE :status_signature_like + """ + ), + { + "query_signature_like": query_like, + "status_signature_like": status_like, + }, + ) + except OperationalError: + return + except Exception: + return + + def _run_compare_write_transaction( engine: Any, operation: Callable[[Any], Any], @@ -9202,6 +11265,8 @@ def _release_invalid_cross_category_recheck_rows( recheck_rows = list((sections or {}).get("amount_mismatch", {}).get("rows", [])) if not recheck_rows: return sections, False + if not ENABLE_GENERATED_RECHECK_CANDIDATES and len(recheck_rows) > 5_000: + return sections, False kept_rows: list[dict[str, Any]] = [] restored_ledger_rows: list[dict[str, Any]] = [] @@ -9259,6 +11324,8 @@ def _repair_invalid_cross_category_matches( sections, changed_scope = _release_invalid_multi_voucher_matches(sections) if not (changed_matched or changed_recheck or changed_scope): return sections + if not ENABLE_GENERATED_RECHECK_CANDIDATES: + return sections return _promote_direct_auto_matches(sections) @@ -10013,11 +12080,6 @@ def _apply_previous_year_erp_candidates( year: int, sections: dict[str, dict[str, Any]], ) -> dict[str, dict[str, Any]]: - # A selected period must only expose and match Hanmac ERP vouchers whose - # draft voucher date belongs to that period. Pulling older ERP candidates - # into the current year made unrelated Hanmac rows appear as current data. - return sections - if year <= 0: return sections ledger_rows = list(sections["ledger_only"]["rows"]) @@ -10025,9 +12087,14 @@ def _apply_previous_year_erp_candidates( return sections available_years = set(_discover_available_fiscal_years(conn)) - prior_years = sorted((candidate_year for candidate_year in available_years if candidate_year < year), reverse=True) - next_years = sorted(candidate_year for candidate_year in available_years if candidate_year > year) - candidate_years = prior_years + next_years + erp_start_year, erp_end_year = get_hanmac_erp_candidate_year_range(year, year) + candidate_years = [ + candidate_year + for candidate_year in (int(erp_start_year or 0), int(erp_end_year or 0)) + if candidate_year + and candidate_year != int(year) + and candidate_year in available_years + ] if not candidate_years: return sections @@ -10204,6 +12271,8 @@ def _apply_previous_year_erp_candidates( return promoted_rows def build_adjacent_review_rows(eligible_ledger_rows: list[dict[str, Any]], *, boundary_phase: bool) -> list[dict[str, Any]]: + if not ENABLE_GENERATED_RECHECK_CANDIDATES: + return [] matched_voucher_keys = remaining_matched_voucher_keys() matched_ledger_keys = remaining_matched_ledger_keys() review_voucher_rows = [ @@ -10241,25 +12310,8 @@ def _apply_previous_year_erp_candidates( review_rows.extend(_build_quality_review_rows(review_ledger_rows, review_voucher_rows, existing_pairs)) return review_rows - # 1) 일반 전표를 먼저 인접 연도 ERP와 비교 - apply_adjacent_candidates(ledger_rows, boundary_phase=False) - normal_review_rows = build_adjacent_review_rows(ledger_rows, boundary_phase=False) - if normal_review_rows: - review_ledger_keys = { - clean(row.get("ledger_row_key")) or build_ledger_row_key(row) - for row in normal_review_rows - } - sections["amount_mismatch"]["rows"].extend(normal_review_rows) - sections["amount_mismatch"]["count"] = len(sections["amount_mismatch"]["rows"]) - sections["ledger_only"]["rows"] = [ - row - for row in sections["ledger_only"]["rows"] - if (clean(row.get("ledger_row_key")) or build_ledger_row_key(row)) not in review_ledger_keys - ] - sections["ledger_only"]["count"] = len(sections["ledger_only"]["rows"]) - ledger_rows = list(sections["ledger_only"]["rows"]) - - # 2) 연초/연말 대체성 전표는 최후 순서로만 인접 연도 ERP와 비교 + # Adjacent-year ERP matching is intentionally limited to boundary-like + # WEHAGO rows; broad cross-year candidate generation is too noisy and slow. apply_adjacent_candidates(ledger_rows, boundary_phase=True) boundary_review_rows = build_adjacent_review_rows(ledger_rows, boundary_phase=True) if boundary_review_rows: @@ -13977,12 +16029,22 @@ def _erp_draft_base(value: Any) -> str: return re.sub(r"-\d+$", "", draft_no) if draft_no else "" +def _row_erp_candidate_year(row: dict[str, Any]) -> int: + draft_year = _erp_draft_year(row.get("draft_no")) + if draft_year: + return draft_year + return int(row.get("voucher_fiscal_year") or row.get("erp_fiscal_year") or row.get("fiscal_year") or 0) + + def _restrict_erp_rows_to_fiscal_year( sections: dict[str, dict[str, Any]], fiscal_year: int, ) -> dict[str, dict[str, Any]]: if fiscal_year <= 0: return sections + erp_start_year, erp_end_year = get_hanmac_erp_candidate_year_range(fiscal_year, fiscal_year) + erp_start_year = int(erp_start_year or fiscal_year) + erp_end_year = int(erp_end_year or fiscal_year) restored_ledger_rows = list(sections.get("ledger_only", {}).get("rows", [])) ledger_keys = { @@ -13991,8 +16053,8 @@ def _restrict_erp_rows_to_fiscal_year( } def keep_row(row: dict[str, Any]) -> bool: - draft_year = _erp_draft_year(row.get("draft_no")) - return not draft_year or draft_year == fiscal_year + candidate_year = _row_erp_candidate_year(row) + return not candidate_year or erp_start_year <= candidate_year <= erp_end_year for status_key in ("matched", "amount_mismatch"): kept_rows: list[dict[str, Any]] = [] @@ -14009,7 +16071,7 @@ def _restrict_erp_rows_to_fiscal_year( continue restored = _blank_voucher_side(row) restored["status_label"] = "Unmatched" - restored["review_reason"] = "ERP_DRAFT_YEAR_OUT_OF_SCOPE" + restored["review_reason"] = "ERP_DRAFT_YEAR_OUT_OF_CANDIDATE_SCOPE" ledger_key = clean(restored.get("ledger_row_key")) or build_ledger_row_key(restored) if ledger_key and ledger_key not in ledger_keys: ledger_keys.add(ledger_key) @@ -14039,6 +16101,7 @@ def _expand_erp_unmatched_groups_from_raw_rows( if start_year is None or end_year is None or not unmatched_groups: return voucher_sections + erp_start_year, erp_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) raw_rows = conn.execute( text( """ @@ -14050,7 +16113,7 @@ def _expand_erp_unmatched_groups_from_raw_rows( ORDER BY fiscal_year ASC, draft_no ASC """ ), - {"start_year": int(start_year), "end_year": int(end_year)}, + {"start_year": int(erp_start_year or start_year), "end_year": int(erp_end_year or end_year)}, ).mappings().all() rows_by_base: dict[tuple[int, str], list[dict[str, Any]]] = {} for source_row in raw_rows: @@ -15121,6 +17184,7 @@ def _raw_erp_amount_entries_by_year( ) -> dict[int, dict[float, list[dict[str, Any]]]]: if start_year is None or end_year is None: return {} + erp_start_year, erp_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) result = conn.execute( text( """ @@ -15131,7 +17195,7 @@ def _raw_erp_amount_entries_by_year( WHERE fiscal_year BETWEEN :start_year AND :end_year """ ), - {"start_year": int(start_year), "end_year": int(end_year)}, + {"start_year": int(erp_start_year or start_year), "end_year": int(erp_end_year or end_year)}, ) rows = result.mappings() if hasattr(result, "mappings") else result indexed: dict[int, dict[float, list[dict[str, Any]]]] = {} @@ -17088,7 +19152,58 @@ def get_status_detail_rows( if cached_response is not None: return cached_response - if normalized_status in QUERY_VOUCHER_STATUS_KEYS and has_voucher_group_filters: + if normalized_status == "hanmac_unconnected": + with engine.begin() as conn: + _ensure_hanmac_unconnected_status_projection(conn, start_year, end_year) + query_page = _load_hanmac_unconnected_compact_projection_page( + conn, + start_year, + end_year, + voucher_filter, + draft_filter, + erp_account_filter, + clean(erp_amount), + erp_vendor_filter, + desc_filter, + safe_offset, + safe_limit, + cursor=cursor, + ) + if query_page is not None: + shown_groups, total_count, next_cursor = query_page + next_offset = safe_offset + len(shown_groups) + shown_rows: list[dict[str, Any]] = [] + for group in shown_groups: + sanitized_rows = _sanitize_voucher_group_rows(list(group.get("rows", []) or [])) + group["rows"] = sanitized_rows + for row in sanitized_rows: + shown_rows.append(dict(row)) + payload = { + "columns": DETAIL_COLUMN_MAP[normalized_status], + "rows": shown_rows, + "groups": shown_groups, + "total_count": total_count, + "shown_count": len(shown_groups), + "offset": safe_offset, + "limit": safe_limit, + "has_more": next_offset < total_count, + "next_offset": next_offset, + "next_cursor": next_cursor, + "notice": "", + "pending": False, + "ready_years": [], + "pending_years": [], + "bank_payable_case_count": 0, + "boundary_excluded_count": 0, + } + return _store_ttl_cached_payload( + _STATUS_DETAIL_RESPONSE_CACHE, + response_cache_key, + payload, + _STATUS_DETAIL_RESPONSE_CACHE_TTL_SEC, + ) + + if normalized_status in QUERY_VOUCHER_STATUS_KEYS and normalized_status not in WEHAGO_FINAL_STATUS_ORDER and has_voucher_group_filters: with engine.begin() as conn: query_page = _load_query_group_page( conn, @@ -17172,7 +19287,104 @@ def get_status_detail_rows( 5.0 if payload.get("pending") else _STATUS_DETAIL_RESPONSE_CACHE_TTL_SEC, ) - if normalized_status in QUERY_VOUCHER_STATUS_KEYS and not has_voucher_group_filters: + if normalized_status in WEHAGO_FINAL_STATUS_ORDER: + with engine.begin() as conn: + final_query_page = _load_final_wehago_status_group_page( + conn, + start_year, + end_year, + normalized_status, + voucher_filter, + draft_filter, + wehago_account_filter, + erp_account_filter, + clean(wehago_amount), + clean(erp_amount), + wehago_vendor_filter, + erp_vendor_filter, + desc_filter, + safe_offset, + safe_limit, + cursor=cursor, + ) + if final_query_page is not None: + shown_groups, total_count, next_cursor = final_query_page + next_offset = safe_offset + len(shown_groups) + shown_rows: list[dict[str, Any]] = [] + for group in shown_groups: + sanitized_rows = _sanitize_voucher_group_rows(list(group.get("rows", []) or [])) + group["rows"] = sanitized_rows + for row in sanitized_rows: + shown_rows.append(dict(row)) + return finalize_status_detail_payload({ + "columns": DETAIL_COLUMN_MAP[normalized_status], + "rows": shown_rows, + "groups": shown_groups, + "total_count": total_count, + "shown_count": len(shown_groups), + "offset": safe_offset, + "limit": safe_limit, + "has_more": next_offset < total_count, + "next_offset": next_offset, + "next_cursor": next_cursor, + "notice": "", + "pending": False, + "ready_years": [], + "pending_years": [], + "bank_payable_case_count": 0, + "boundary_excluded_count": 0, + }) + + if normalized_status in COMPACT_EXPORT_STATUS_KEYS: + with engine.begin() as conn: + _ensure_export_compact_status_projection(conn, start_year, end_year, normalized_status) + compact_page = _load_compact_status_projection_page( + conn, + start_year, + end_year, + normalized_status, + voucher_filter, + draft_filter, + wehago_account_filter, + erp_account_filter, + clean(wehago_amount), + clean(erp_amount), + wehago_vendor_filter, + erp_vendor_filter, + desc_filter, + safe_offset, + safe_limit, + cursor=cursor, + ) + if compact_page is not None: + shown_groups, total_count, next_cursor = compact_page + next_offset = safe_offset + len(shown_groups) + shown_rows: list[dict[str, Any]] = [] + for group in shown_groups: + sanitized_rows = _sanitize_voucher_group_rows(list(group.get("rows", []) or [])) + group["rows"] = sanitized_rows + for row in sanitized_rows: + shown_rows.append(dict(row)) + return finalize_status_detail_payload({ + "columns": DETAIL_COLUMN_MAP[normalized_status], + "rows": shown_rows, + "groups": shown_groups, + "total_count": total_count, + "shown_count": len(shown_groups), + "offset": safe_offset, + "limit": safe_limit, + "has_more": next_offset < total_count, + "next_offset": next_offset, + "next_cursor": next_cursor, + "notice": "", + "pending": False, + "ready_years": [], + "pending_years": [], + "bank_payable_case_count": 0, + "boundary_excluded_count": 0, + }) + + if normalized_status in QUERY_VOUCHER_STATUS_KEYS and normalized_status != "hanmac_unconnected" and not has_voucher_group_filters: direct_query_page = _fast_sqlite_query_group_page( engine, start_year, @@ -17211,7 +19423,7 @@ def get_status_detail_rows( "boundary_excluded_count": 0, }) - if normalized_status in QUERY_VOUCHER_STATUS_KEYS: + if normalized_status in QUERY_VOUCHER_STATUS_KEYS and normalized_status != "hanmac_unconnected": persisted_page_payload: dict[str, Any] | None = None if not has_voucher_group_filters: with engine.begin() as conn: @@ -17999,7 +20211,7 @@ def _status_detail_response_cache_key( cursor: str, ) -> str: normalized = { - "logic_version": QUERY_PROJECTION_VERSION, + "logic_version": f"{QUERY_PROJECTION_VERSION}|final-detail-v2", "start_year": int(start_year or 0), "end_year": int(end_year or 0), "status": clean(status).lower(), @@ -19081,6 +21293,20 @@ def _hanmac_voucher_base(draft_no: Any) -> str: return re.sub(r"-\d+$", "", clean(draft_no)) +def _hanmac_voucher_date_from_numbers(*values: Any) -> str: + for value in values: + match = re.search(r"(20\d{2})(\d{2})(\d{2})", clean(value)) + if not match: + continue + year, month, day = match.groups() + try: + parsed = date(int(year), int(month), int(day)) + except ValueError: + continue + return parsed.isoformat() + return "" + + def _is_hanmac_cost_account(account_code: Any, account_name: Any) -> bool: family = _classify_account_family(account_code, account_name) if _is_vat_family(family) or family in {"bank", "payable", "receivable", "advance"}: @@ -19097,7 +21323,14 @@ def _hanmac_unconnected_cache_key( f"{year}:{_get_fast_year_export_row_cache_signature(conn, year)}" for year in range(int(start_year), int(end_year) + 1) ] - return "|".join([QUERY_PROJECTION_VERSION, str(start_year), str(end_year), *signatures]) + return "|".join( + [ + QUERY_PROJECTION_VERSION, + str(start_year), + str(end_year), + *signatures, + ] + ) def _matched_hanmac_cost_voucher_bases( @@ -19113,27 +21346,22 @@ def _matched_hanmac_cost_voucher_bases( rows = conn.execute( text( """ - SELECT fiscal_year, draft_no, voucher_account_name + SELECT fiscal_year, draft_no, group_draft_no, voucher_account_name, ledger_account_name FROM wehago_compare_export_row_cache WHERE fiscal_year = :fiscal_year AND snapshot_signature = :signature AND status_key IN ('voucher_matched', 'erp_voucher_matched', 'voucher_recheck') - AND COALESCE(draft_no, '') <> '' AND COALESCE(voucher_account_name, '') <> '' - AND ( - COALESCE(ledger_account_name, '') <> '' - OR COALESCE(ledger_desc, '') <> '' - ) + AND COALESCE(ledger_account_name, '') <> '' """ ), {"fiscal_year": year, "signature": signature}, ).mappings().all() for row in rows: - if not _is_hanmac_cost_account("", row.get("voucher_account_name")): - continue - voucher_base = _hanmac_voucher_base(row.get("draft_no")) + voucher_base = _hanmac_voucher_base(row.get("draft_no")) or _hanmac_voucher_base(row.get("group_draft_no")) if voucher_base: - matched_bases.add((int(row.get("fiscal_year") or year), voucher_base)) + draft_year = _erp_draft_year(row.get("draft_no")) or _erp_draft_year(row.get("group_draft_no")) + matched_bases.add((draft_year or int(row.get("fiscal_year") or year), voucher_base)) return matched_bases @@ -19153,7 +21381,7 @@ def _load_hanmac_unconnected_source_groups( if cached and (now - float(cached.get("ts", 0))) <= _HANMAC_UNCONNECTED_CACHE_TTL_SEC: return cached["groups"] - matched_cost_bases = _matched_hanmac_cost_voucher_bases(conn, start_year, end_year) + matched_bases = _matched_hanmac_cost_voucher_bases(conn, start_year, end_year) raw_rows = conn.execute( text( """ @@ -19171,15 +21399,20 @@ def _load_hanmac_unconnected_source_groups( for source_row in raw_rows: row = dict(source_row) fiscal_year = int(row.get("fiscal_year") or 0) - voucher_base = _hanmac_voucher_base(row.get("draft_no")) + voucher_base = _hanmac_voucher_base(row.get("draft_no")) or _hanmac_voucher_base(row.get("confirmed_no")) draft_year = _erp_draft_year(row.get("draft_no")) - if fiscal_year <= 0 or (draft_year and draft_year != fiscal_year) or not voucher_base: + if fiscal_year <= 0 or not voucher_base: continue - key = (fiscal_year, voucher_base) + if draft_year and not (int(start_year) <= draft_year <= int(end_year)): + continue + key_year = draft_year or fiscal_year + if not (int(start_year) <= key_year <= int(end_year)): + continue + key = (key_year, voucher_base) current = grouped.setdefault( key, { - "fiscal_year": fiscal_year, + "fiscal_year": key_year, "status_label": "Hanmac unconnected", "ledger_date": "", "proof_date": "", @@ -19195,13 +21428,13 @@ def _load_hanmac_unconnected_source_groups( "voucher_accounts": [], "ledger_vendors": "", "voucher_vendors": [], - "review_reason": "비용 계정 WEHAGO 미연결", - "has_cost_account": False, + "review_reason": "WEHAGO 매치 계정 없음", "voucher_nos": [], + "draft_nos": [], "rows": [], }, ) - proof_date = clean(row.get("proof_date")) + proof_date = clean(row.get("proof_date")) or _hanmac_voucher_date_from_numbers(row.get("draft_no"), row.get("confirmed_no")) confirmed_no = clean(row.get("confirmed_no")) account_code = clean(row.get("account_code")) account_name = clean(row.get("account_name")) @@ -19213,22 +21446,23 @@ def _load_hanmac_unconnected_source_groups( current["proof_date"] = proof_date if confirmed_no: append_distinct(current["voucher_nos"], confirmed_no) + append_distinct(current["draft_nos"], row.get("draft_no")) append_distinct(current["voucher_accounts"], account_name) append_distinct(current["voucher_vendors"], voucher_vendor) current["voucher_row_count"] += 1 current["voucher_debit"] += voucher_debit current["voucher_credit"] += voucher_credit - if _is_hanmac_cost_account(account_code, account_name): - current["has_cost_account"] = True current["rows"].append( { - "fiscal_year": fiscal_year, + "fiscal_year": key_year, "status_label": "Hanmac unconnected", "ledger_date": "", "proof_date": proof_date, "voucher_no": confirmed_no, "draft_no": clean(row.get("draft_no")), + "ledger_account_code": "", "ledger_account_name": "", + "voucher_account_code": account_code, "voucher_account_name": account_name, "ledger_vendor": "", "voucher_vendor": voucher_vendor, @@ -19243,12 +21477,24 @@ def _load_hanmac_unconnected_source_groups( groups: list[dict[str, Any]] = [] for key, current in grouped.items(): - if not current.pop("has_cost_account") or key in matched_cost_bases: + if key in matched_bases: continue current["voucher_no"] = ", ".join(current.pop("voucher_nos")) + current["draft_no"] = ", ".join(current.pop("draft_nos")) or current["draft_no"] current["voucher_accounts"] = ", ".join(current["voucher_accounts"]) current["voucher_vendors"] = ", ".join(current["voucher_vendors"]) - groups.append({"summary": current, "rows": list(current.pop("rows"))}) + rows = list(current.pop("rows")) + rows.sort( + key=lambda row: ( + clean(row.get("proof_date")), + clean(row.get("draft_no")), + clean(row.get("voucher_no")), + clean(row.get("voucher_account_name")), + -parse_amount(row.get("voucher_debit")), + -parse_amount(row.get("voucher_credit")), + ) + ) + groups.append({"summary": current, "rows": rows}) groups.sort( key=lambda group: ( int(group.get("summary", {}).get("fiscal_year") or 0), @@ -19261,6 +21507,690 @@ def _load_hanmac_unconnected_source_groups( return groups +def _ensure_hanmac_unconnected_status_projection( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> str: + if start_year is None or end_year is None: + return "" + signature = _hanmac_unconnected_status_projection_signature(conn, start_year, end_year) + if not signature: + return "" + active_signature = _active_status_projection_signature(conn, "hanmac_unconnected", start_year, end_year) + if active_signature == signature: + return signature + existing_count = conn.execute( + text( + """ + SELECT COUNT(*) + FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = 'hanmac_unconnected' + AND signature = :signature + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ).scalar_one() + if int(existing_count or 0) > 0: + _store_active_status_projection_signature( + conn, + "hanmac_unconnected", + start_year, + end_year, + signature, + int(existing_count or 0), + ) + return signature + groups = _load_hanmac_unconnected_source_groups(conn, int(start_year), int(end_year)) + _store_hanmac_unconnected_compact_projection( + conn, + start_year, + end_year, + signature, + groups, + ) + return signature + + +def _store_hanmac_unconnected_compact_projection( + conn: Any, + start_year: int | None, + end_year: int | None, + signature: str, + groups: list[dict[str, Any]], +) -> None: + if start_year is None or end_year is None or not clean(signature): + return + conn.execute( + text( + """ + DELETE FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = 'hanmac_unconnected' + AND signature = :signature + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": clean(signature)}, + ) + payloads: list[dict[str, Any]] = [] + for group_index, group in enumerate(groups): + summary = dict(group.get("summary", {}) or {}) + rows = _sanitize_voucher_group_rows(list(group.get("rows", []) or [])) + search_text = " ".join( + filter( + None, + [ + clean(summary.get("voucher_no")), + clean(summary.get("draft_no")), + clean(summary.get("voucher_accounts")), + clean(summary.get("voucher_vendors")), + clean(summary.get("review_reason")), + " ".join(clean(row.get("voucher_desc")) for row in rows), + ], + ) + ) + payloads.append( + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": "hanmac_unconnected", + "signature": clean(signature), + "group_index": group_index, + "fiscal_year": int(summary.get("fiscal_year") or 0), + "ledger_date": clean(summary.get("ledger_date")), + "proof_date": clean(summary.get("proof_date")), + "voucher_no": clean(summary.get("voucher_no")), + "draft_no": clean(summary.get("draft_no")), + "ledger_row_count": int(summary.get("ledger_row_count") or 0), + "voucher_row_count": int(summary.get("voucher_row_count") or 0), + "ledger_debit": parse_amount(summary.get("ledger_debit")), + "ledger_credit": parse_amount(summary.get("ledger_credit")), + "voucher_debit": parse_amount(summary.get("voucher_debit")), + "voucher_credit": parse_amount(summary.get("voucher_credit")), + "ledger_accounts": clean(summary.get("ledger_accounts")), + "voucher_accounts": clean(summary.get("voucher_accounts")), + "ledger_vendors": clean(summary.get("ledger_vendors")), + "voucher_vendors": clean(summary.get("voucher_vendors")), + "review_reason": clean(summary.get("review_reason")), + "search_text": search_text, + "summary_json": json.dumps(summary, ensure_ascii=False, separators=(",", ":")), + "rows_json": json.dumps(rows, ensure_ascii=False, separators=(",", ":")), + } + ) + insert_sql = text( + """ + INSERT INTO wehago_status_projection_groups ( + start_year, end_year, status_key, signature, group_index, + fiscal_year, ledger_date, proof_date, voucher_no, draft_no, + ledger_row_count, voucher_row_count, + ledger_debit, ledger_credit, voucher_debit, voucher_credit, + ledger_accounts, voucher_accounts, ledger_vendors, voucher_vendors, + review_reason, search_text, summary_json, rows_json, created_at, updated_at + ) VALUES ( + :start_year, :end_year, :status_key, :signature, :group_index, + :fiscal_year, :ledger_date, :proof_date, :voucher_no, :draft_no, + :ledger_row_count, :voucher_row_count, + :ledger_debit, :ledger_credit, :voucher_debit, :voucher_credit, + :ledger_accounts, :voucher_accounts, :ledger_vendors, :voucher_vendors, + :review_reason, :search_text, :summary_json, :rows_json, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ) + for chunk in _chunked(payloads): + conn.execute(insert_sql, chunk) + _store_active_status_projection_signature( + conn, + "hanmac_unconnected", + start_year, + end_year, + signature, + len(groups), + ) + + +def _load_hanmac_unconnected_status_projection_count( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> int | None: + if start_year is None or end_year is None: + return None + signature = _active_status_projection_signature(conn, "hanmac_unconnected", start_year, end_year) + if not signature: + signature = _ensure_hanmac_unconnected_status_projection(conn, start_year, end_year) + if not signature: + return None + count = conn.execute( + text( + """ + SELECT COUNT(*) + FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = 'hanmac_unconnected' + AND signature = :signature + """ + ), + {"start_year": int(start_year), "end_year": int(end_year), "signature": signature}, + ).scalar_one() + return int(count or 0) + + +def _load_hanmac_unconnected_compact_projection_page( + conn: Any, + start_year: int | None, + end_year: int | None, + voucher_no: str, + draft_no: str, + erp_account: str, + erp_amount: str, + erp_vendor: str, + desc_keyword: str, + offset: int, + limit: int, + cursor: str = "", +) -> tuple[list[dict[str, Any]], int, str] | None: + if start_year is None or end_year is None: + return None + signature = _active_status_projection_signature(conn, "hanmac_unconnected", start_year, end_year) + if not signature: + return None + voucher_filter = clean(voucher_no) + draft_filter = clean(draft_no) + erp_account_filter = normalize_text(erp_account) + erp_vendor_filter = normalize_text(erp_vendor) + desc_filter = normalize_text(desc_keyword) + erp_amount_filter = clean(erp_amount) + params = { + "start_year": int(start_year), + "end_year": int(end_year), + "signature": signature, + "voucher_keyword": voucher_filter, + "voucher_like": f"%{voucher_filter}%", + "draft_keyword": draft_filter, + "draft_like": f"%{draft_filter}%", + "erp_account_keyword": erp_account_filter, + "erp_account_like": f"%{erp_account_filter}%", + "erp_vendor_keyword": erp_vendor_filter, + "erp_vendor_like": f"%{erp_vendor_filter}%", + "desc_keyword": desc_filter, + "desc_like": f"%{desc_filter}%", + "erp_amount_keyword": erp_amount_filter, + "limit": int(limit), + "offset": int(offset), + } + where_sql = """ + start_year = :start_year + AND end_year = :end_year + AND status_key = 'hanmac_unconnected' + AND signature = :signature + AND fiscal_year BETWEEN :start_year AND :end_year + AND (:voucher_keyword = '' OR COALESCE(voucher_no, '') LIKE :voucher_like) + AND (:draft_keyword = '' OR COALESCE(draft_no, '') LIKE :draft_like) + AND (:erp_account_keyword = '' OR COALESCE(voucher_accounts, '') LIKE :erp_account_like) + AND (:erp_vendor_keyword = '' OR COALESCE(voucher_vendors, '') LIKE :erp_vendor_like) + AND (:desc_keyword = '' OR COALESCE(search_text, '') LIKE :desc_like) + AND ( + :erp_amount_keyword = '' + OR CAST(ABS(COALESCE(voucher_debit, 0)) AS TEXT) LIKE '%' || :erp_amount_keyword || '%' + OR CAST(ABS(COALESCE(voucher_credit, 0)) AS TEXT) LIKE '%' || :erp_amount_keyword || '%' + ) + """ + total_count = int( + conn.execute( + text(f"SELECT COUNT(*) FROM wehago_status_projection_groups WHERE {where_sql}"), + params, + ).scalar_one() + or 0 + ) + if total_count <= 0: + return [], 0, "" + cursor_year = 0 + cursor_group_index = -1 + cursor_text = clean(cursor) + if cursor_text: + try: + raw_year, raw_group = cursor_text.split(":", 1) + cursor_year = int(raw_year or 0) + cursor_group_index = int(raw_group or -1) + except Exception: + cursor_year = 0 + cursor_group_index = -1 + cursor_text = "" + query_sql = f"SELECT * FROM wehago_status_projection_groups WHERE {where_sql}" + if cursor_text: + query_sql += """ + AND ( + fiscal_year > :cursor_year + OR (fiscal_year = :cursor_year AND group_index > :cursor_group_index) + ) + """ + params["cursor_year"] = cursor_year + params["cursor_group_index"] = cursor_group_index + query_sql += " ORDER BY fiscal_year ASC, group_index ASC LIMIT :limit" + if not cursor_text: + query_sql += " OFFSET :offset" + rows = conn.execute(text(query_sql), params).mappings().all() + groups: list[dict[str, Any]] = [] + for row in rows: + try: + summary = json.loads(clean(row.get("summary_json")) or "{}") + except Exception: + summary = {} + try: + detail_rows = json.loads(clean(row.get("rows_json")) or "[]") + except Exception: + detail_rows = [] + if not isinstance(summary, dict): + summary = {} + if not isinstance(detail_rows, list): + detail_rows = [] + groups.append({"summary": summary, "rows": _sanitize_voucher_group_rows(detail_rows)}) + last_row = rows[-1] + next_cursor = f"{int(last_row.get('fiscal_year') or 0)}:{int(last_row.get('group_index') or 0)}" if len(rows) >= int(limit or 0) else "" + return groups, total_count, next_cursor + + +def _store_export_compact_status_projection( + conn: Any, + start_year: int | None, + end_year: int | None, + status_key: str, + signature: str, + groups: list[dict[str, Any]], +) -> None: + if start_year is None or end_year is None or clean(status_key) not in COMPACT_EXPORT_STATUS_KEYS or not clean(signature): + return + normalized_status = clean(status_key) + conn.execute( + text( + """ + DELETE FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": clean(signature), + }, + ) + payloads: list[dict[str, Any]] = [] + for group_index, group in enumerate(groups): + summary = dict(group.get("summary", {}) or {}) + rows = _sanitize_voucher_group_rows(list(group.get("rows", []) or [])) + search_text = " ".join( + filter( + None, + [ + clean(summary.get("voucher_no")), + clean(summary.get("draft_no")), + clean(summary.get("ledger_accounts")), + clean(summary.get("voucher_accounts")), + clean(summary.get("ledger_vendors")), + clean(summary.get("voucher_vendors")), + clean(summary.get("review_reason")), + " ".join(clean(row.get("ledger_desc")) for row in rows), + " ".join(clean(row.get("voucher_desc")) for row in rows), + ], + ) + ) + payloads.append( + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": clean(signature), + "group_index": group_index, + "fiscal_year": int(summary.get("fiscal_year") or 0), + "ledger_date": clean(summary.get("ledger_date")), + "proof_date": clean(summary.get("proof_date")), + "voucher_no": clean(summary.get("voucher_no")), + "draft_no": clean(summary.get("draft_no")), + "ledger_row_count": int(summary.get("ledger_row_count") or 0), + "voucher_row_count": int(summary.get("voucher_row_count") or 0), + "ledger_debit": parse_amount(summary.get("ledger_debit")), + "ledger_credit": parse_amount(summary.get("ledger_credit")), + "voucher_debit": parse_amount(summary.get("voucher_debit")), + "voucher_credit": parse_amount(summary.get("voucher_credit")), + "ledger_accounts": clean(summary.get("ledger_accounts")), + "voucher_accounts": clean(summary.get("voucher_accounts")), + "ledger_vendors": clean(summary.get("ledger_vendors")), + "voucher_vendors": clean(summary.get("voucher_vendors")), + "review_reason": clean(summary.get("review_reason")), + "search_text": search_text, + "summary_json": json.dumps(summary, ensure_ascii=False, separators=(",", ":")), + "rows_json": json.dumps(rows, ensure_ascii=False, separators=(",", ":")), + } + ) + insert_sql = text( + """ + INSERT INTO wehago_status_projection_groups ( + start_year, end_year, status_key, signature, group_index, + fiscal_year, ledger_date, proof_date, voucher_no, draft_no, + ledger_row_count, voucher_row_count, + ledger_debit, ledger_credit, voucher_debit, voucher_credit, + ledger_accounts, voucher_accounts, ledger_vendors, voucher_vendors, + review_reason, search_text, summary_json, rows_json, created_at, updated_at + ) VALUES ( + :start_year, :end_year, :status_key, :signature, :group_index, + :fiscal_year, :ledger_date, :proof_date, :voucher_no, :draft_no, + :ledger_row_count, :voucher_row_count, + :ledger_debit, :ledger_credit, :voucher_debit, :voucher_credit, + :ledger_accounts, :voucher_accounts, :ledger_vendors, :voucher_vendors, + :review_reason, :search_text, :summary_json, :rows_json, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ) + for chunk in _chunked(payloads): + conn.execute(insert_sql, chunk) + _store_active_status_projection_signature( + conn, + normalized_status, + start_year, + end_year, + signature, + len(groups), + ) + + +def _ensure_export_compact_status_projection( + conn: Any, + start_year: int | None, + end_year: int | None, + status_key: str, +) -> str: + normalized_status = clean(status_key) + if start_year is None or end_year is None or normalized_status not in COMPACT_EXPORT_STATUS_KEYS: + return "" + signature = _status_projection_signature(conn, normalized_status, start_year, end_year) + if not signature: + return "" + active_signature = _active_status_projection_signature(conn, normalized_status, start_year, end_year) + if active_signature == signature: + return signature + existing_count = conn.execute( + text( + """ + SELECT COUNT(*) + FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": signature, + }, + ).scalar_one() + if int(existing_count or 0) > 0: + _store_active_status_projection_signature( + conn, + normalized_status, + start_year, + end_year, + signature, + int(existing_count or 0), + ) + return signature + export_rows, _row_count = _iter_cached_export_rows( + conn, + start_year, + end_year, + normalized_status, + "", + "", + "", + "", + "", + "", + "", + "", + "", + ) + groups = _build_voucher_groups_from_cached_export_rows(export_rows, normalized_status) + _store_export_compact_status_projection( + conn, + start_year, + end_year, + normalized_status, + signature, + groups, + ) + return signature + + +def _load_compact_status_projection_count( + conn: Any, + start_year: int | None, + end_year: int | None, + status_key: str, +) -> int | None: + normalized_status = clean(status_key) + if start_year is None or end_year is None or normalized_status not in COMPACT_EXPORT_STATUS_KEYS: + return None + signature = _active_status_projection_signature(conn, normalized_status, start_year, end_year) + if not signature: + signature = _ensure_export_compact_status_projection(conn, start_year, end_year, normalized_status) + if not signature: + return None + count = conn.execute( + text( + """ + SELECT COUNT(*) + FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + """ + ), + { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": signature, + }, + ).scalar_one() + return int(count or 0) + + +def _load_active_status_projection_counts_for_range( + conn: Any, + start_year: int | None, + end_year: int | None, + status_keys: Iterable[str], +) -> dict[str, int]: + if start_year is None or end_year is None: + return {} + active_signatures = _active_status_projection_signatures_for_range( + conn, + start_year, + end_year, + status_keys, + ) + if not active_signatures: + return {} + status_placeholders = ", ".join(f":status_{index}" for index, _status in enumerate(active_signatures)) + params: dict[str, Any] = { + "start_year": int(start_year), + "end_year": int(end_year), + **{f"status_{index}": status for index, status in enumerate(active_signatures)}, + } + signature_conditions: list[str] = [] + for index, (status_key, signature) in enumerate(active_signatures.items()): + params[f"signature_{index}"] = signature + signature_conditions.append(f"(status_key = :status_{index} AND signature = :signature_{index})") + rows = conn.execute( + text( + f""" + SELECT status_key, COUNT(*) AS count + FROM wehago_status_projection_groups + WHERE start_year = :start_year + AND end_year = :end_year + AND status_key IN ({status_placeholders}) + AND ({" OR ".join(signature_conditions)}) + GROUP BY status_key + """ + ), + params, + ).mappings().all() + return {clean(row.get("status_key")): int(row.get("count") or 0) for row in rows} + + +def _all_voucher_status_projections_ready( + conn: Any, + start_year: int | None, + end_year: int | None, +) -> bool: + if start_year is None or end_year is None: + return False + required_statuses = {"hanmac_unconnected", *COMPACT_EXPORT_STATUS_KEYS} + active = _active_status_projection_signatures_for_range(conn, start_year, end_year, required_statuses) + return required_statuses.issubset(set(active)) + + +def _load_compact_status_projection_page( + conn: Any, + start_year: int | None, + end_year: int | None, + status_key: str, + voucher_no: str, + draft_no: str, + wehago_account: str, + erp_account: str, + wehago_amount: str, + erp_amount: str, + wehago_vendor: str, + erp_vendor: str, + desc_keyword: str, + offset: int, + limit: int, + cursor: str = "", +) -> tuple[list[dict[str, Any]], int, str] | None: + normalized_status = clean(status_key) + if start_year is None or end_year is None or normalized_status not in COMPACT_EXPORT_STATUS_KEYS: + return None + signature = _active_status_projection_signature(conn, normalized_status, start_year, end_year) + if not signature: + return None + voucher_filter = clean(voucher_no) + draft_filter = clean(draft_no) + wehago_account_filter = normalize_text(wehago_account) + erp_account_filter = normalize_text(erp_account) + wehago_vendor_filter = normalize_text(wehago_vendor) + erp_vendor_filter = normalize_text(erp_vendor) + desc_filter = normalize_text(desc_keyword) + wehago_amount_filter = clean(wehago_amount) + erp_amount_filter = clean(erp_amount) + params = { + "start_year": int(start_year), + "end_year": int(end_year), + "status_key": normalized_status, + "signature": signature, + "voucher_keyword": voucher_filter, + "voucher_like": f"%{voucher_filter}%", + "draft_keyword": draft_filter, + "draft_like": f"%{draft_filter}%", + "wehago_account_keyword": wehago_account_filter, + "wehago_account_like": f"%{wehago_account_filter}%", + "erp_account_keyword": erp_account_filter, + "erp_account_like": f"%{erp_account_filter}%", + "wehago_vendor_keyword": wehago_vendor_filter, + "wehago_vendor_like": f"%{wehago_vendor_filter}%", + "erp_vendor_keyword": erp_vendor_filter, + "erp_vendor_like": f"%{erp_vendor_filter}%", + "desc_keyword": desc_filter, + "desc_like": f"%{desc_filter}%", + "wehago_amount_keyword": wehago_amount_filter, + "erp_amount_keyword": erp_amount_filter, + "limit": int(limit), + "offset": int(offset), + } + where_sql = """ + start_year = :start_year + AND end_year = :end_year + AND status_key = :status_key + AND signature = :signature + AND fiscal_year BETWEEN :start_year AND :end_year + AND (:voucher_keyword = '' OR COALESCE(voucher_no, '') LIKE :voucher_like) + AND (:draft_keyword = '' OR COALESCE(draft_no, '') LIKE :draft_like) + AND (:wehago_account_keyword = '' OR COALESCE(ledger_accounts, '') LIKE :wehago_account_like) + AND (:erp_account_keyword = '' OR COALESCE(voucher_accounts, '') LIKE :erp_account_like) + AND (:wehago_vendor_keyword = '' OR COALESCE(ledger_vendors, '') LIKE :wehago_vendor_like) + AND (:erp_vendor_keyword = '' OR COALESCE(voucher_vendors, '') LIKE :erp_vendor_like) + AND (:desc_keyword = '' OR COALESCE(search_text, '') LIKE :desc_like) + AND ( + :wehago_amount_keyword = '' + OR CAST(ABS(COALESCE(ledger_debit, 0)) AS TEXT) LIKE '%' || :wehago_amount_keyword || '%' + OR CAST(ABS(COALESCE(ledger_credit, 0)) AS TEXT) LIKE '%' || :wehago_amount_keyword || '%' + ) + AND ( + :erp_amount_keyword = '' + OR CAST(ABS(COALESCE(voucher_debit, 0)) AS TEXT) LIKE '%' || :erp_amount_keyword || '%' + OR CAST(ABS(COALESCE(voucher_credit, 0)) AS TEXT) LIKE '%' || :erp_amount_keyword || '%' + ) + """ + total_count = int(conn.execute(text(f"SELECT COUNT(*) FROM wehago_status_projection_groups WHERE {where_sql}"), params).scalar_one() or 0) + if total_count <= 0: + return [], 0, "" + cursor_year = 0 + cursor_group_index = -1 + cursor_text = clean(cursor) + if cursor_text: + try: + raw_year, raw_group = cursor_text.split(":", 1) + cursor_year = int(raw_year or 0) + cursor_group_index = int(raw_group or -1) + except Exception: + cursor_year = 0 + cursor_group_index = -1 + cursor_text = "" + query_sql = f"SELECT * FROM wehago_status_projection_groups WHERE {where_sql}" + if cursor_text: + query_sql += """ + AND ( + fiscal_year > :cursor_year + OR (fiscal_year = :cursor_year AND group_index > :cursor_group_index) + ) + """ + params["cursor_year"] = cursor_year + params["cursor_group_index"] = cursor_group_index + query_sql += " ORDER BY fiscal_year ASC, group_index ASC LIMIT :limit" + if not cursor_text: + query_sql += " OFFSET :offset" + rows = conn.execute(text(query_sql), params).mappings().all() + groups: list[dict[str, Any]] = [] + for row in rows: + try: + summary = json.loads(clean(row.get("summary_json")) or "{}") + except Exception: + summary = {} + try: + detail_rows = json.loads(clean(row.get("rows_json")) or "[]") + except Exception: + detail_rows = [] + if not isinstance(summary, dict): + summary = {} + if not isinstance(detail_rows, list): + detail_rows = [] + groups.append({"summary": summary, "rows": _sanitize_voucher_group_rows(detail_rows)}) + last_row = rows[-1] + next_cursor = f"{int(last_row.get('fiscal_year') or 0)}:{int(last_row.get('group_index') or 0)}" if len(rows) >= int(limit or 0) else "" + return groups, total_count, next_cursor + + def _count_hanmac_unconnected_export_groups(conn: Any, fiscal_year: int, signature: str) -> int: return len(_load_hanmac_unconnected_source_groups(conn, fiscal_year, fiscal_year)) @@ -20174,12 +23104,9 @@ def get_wehago_compare_dashboard( latest_upload = fetch_latest_upload_meta(conn) last_action = get_last_action_summary(conn=conn) metric_counts = {status_key: 0 for status_key, _, _ in STATUS_META} + pending = False if include_metric_counts: - if warm_caches: - with engine.begin() as conn: - metric_counts = get_dashboard_metric_counts(conn, start_year, end_year) - else: - metric_counts, _pending = get_dashboard_metric_counts_nonblocking(engine, start_year, end_year) + metric_counts, pending = get_dashboard_metric_counts_nonblocking(engine, start_year, end_year) metric_sections = [ { "key": status_key, @@ -20191,10 +23118,24 @@ def get_wehago_compare_dashboard( } for status_key, label, description in STATUS_META ] - if warm_caches: + compact_status_ready = False + with engine.begin() as conn: + compact_status_ready = _all_voucher_status_projections_ready(conn, start_year, end_year) + if compact_status_ready: + pending = False + if warm_caches and not compact_status_ready: warm_metric_counts_async(engine, start_year, end_year) if start_year == end_year: warm_status_cache_async(engine, start_year, end_year) + with engine.begin() as conn: + final_status_summary = get_wehago_final_status_summary_from_conn( + conn, + start_year, + end_year, + metric_counts, + ) + metric_sections = apply_wehago_final_status_counts_to_metric_sections(metric_sections, final_status_summary) + erp_candidate_start_year, erp_candidate_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) return { "page_title": "현황", @@ -20203,7 +23144,11 @@ def get_wehago_compare_dashboard( "selected_start_year": start_year, "selected_end_year": end_year, "available_years": years, + "hanmac_erp_candidate_start_year": erp_candidate_start_year, + "hanmac_erp_candidate_end_year": erp_candidate_end_year, "metric_sections": metric_sections, + "pending": pending, + "wehago_final_status_summary": final_status_summary, "wehago_account_options": account_options["wehago"], "erp_account_options": account_options["erp"], "latest_upload": latest_upload, @@ -20254,7 +23199,6 @@ def get_wehago_compare_summary( if start_year and end_year and start_year > end_year: start_year, end_year = end_year, start_year - counts = _load_broader_query_projection_counts_from_groups(engine, start_year, end_year) or _empty_metric_counts() last_action_row = sqlite_conn.execute( """ SELECT id, action_type, payload_json, created_at @@ -20293,6 +23237,11 @@ def get_wehago_compare_summary( snapshot_state.setdefault(state, []) snapshot_state[state].append(year) pending = bool(snapshot_state.get("missing") or snapshot_state.get("stale") or snapshot_state.get("queued") or snapshot_state.get("running")) + counts, count_pending = get_dashboard_metric_counts_nonblocking(engine, start_year, end_year) + pending = pending or count_pending + with engine.begin() as conn: + if _all_voucher_status_projections_ready(conn, start_year, end_year): + pending = False metric_sections = [ { "key": status_key, @@ -20304,6 +23253,15 @@ def get_wehago_compare_summary( } for status_key, label, description in STATUS_META ] + with engine.begin() as conn: + final_status_summary = get_wehago_final_status_summary_from_conn( + conn, + start_year, + end_year, + counts, + ) + metric_sections = apply_wehago_final_status_counts_to_metric_sections(metric_sections, final_status_summary) + erp_candidate_start_year, erp_candidate_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) snapshot_aggregate = { "ready_count": len(snapshot_state.get("ready", [])), "stale_count": len(snapshot_state.get("stale", [])), @@ -20315,7 +23273,10 @@ def get_wehago_compare_summary( return { "selected_start_year": start_year, "selected_end_year": end_year, + "hanmac_erp_candidate_start_year": erp_candidate_start_year, + "hanmac_erp_candidate_end_year": erp_candidate_end_year, "metric_sections": metric_sections, + "wehago_final_status_summary": final_status_summary, "last_action": last_action, "pending": pending, "snapshot_state": snapshot_state, @@ -20391,6 +23352,9 @@ def get_wehago_compare_summary( start_year, end_year, ) + with engine.begin() as conn: + if _all_voucher_status_projections_ready(conn, start_year, end_year): + pending = False metric_sections = [ { "key": status_key, @@ -20402,10 +23366,22 @@ def get_wehago_compare_summary( } for status_key, label, description in STATUS_META ] + with engine.begin() as conn: + final_status_summary = get_wehago_final_status_summary_from_conn( + conn, + start_year, + end_year, + metric_counts, + ) + metric_sections = apply_wehago_final_status_counts_to_metric_sections(metric_sections, final_status_summary) + erp_candidate_start_year, erp_candidate_end_year = get_hanmac_erp_candidate_year_range(start_year, end_year) payload = { "selected_start_year": start_year, "selected_end_year": end_year, + "hanmac_erp_candidate_start_year": erp_candidate_start_year, + "hanmac_erp_candidate_end_year": erp_candidate_end_year, "metric_sections": metric_sections, + "wehago_final_status_summary": final_status_summary, "last_action": last_action, "pending": pending, "snapshot_state": snapshot_state,