from __future__ import annotations import argparse import json import re import sqlite3 from collections import Counter, defaultdict from datetime import datetime from pathlib import Path from typing import Any import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from runtime_config import DB_PATH from wehago_compare import ( QUERY_PROJECTION_VERSION, _account_category_pair_allowed, _classify_account_category, _classify_account_family, _is_wehago_excepted_voucher_group, _is_vat_family, _move_wehago_confirmed_reversal_pairs_to_excepted, _move_wehago_offset_tax_invoice_groups_to_excepted, _nature_compatible, build_voucher_row_key, clean, ) YEAR = 2025 WEHAGO_STATUSES = ("voucher_matched", "voucher_unmatched", "voucher_recheck", "voucher_excepted") ERP_STATUSES = ("erp_voucher_matched", "erp_voucher_unmatched") ALL_VOUCHER_STATUSES = WEHAGO_STATUSES + ERP_STATUSES RAW_ERP_ROWS_BY_DRAFT_BASE: dict[str, list[dict[str, Any]]] = defaultdict(list) MANUAL_OFFSET_EXCEPTED_IDENTITIES: set[str] = set() GROUP_COLUMNS = ( "start_year", "end_year", "status_key", "signature", "group_index", "fiscal_year", "ledger_date", "proof_date", "voucher_no", "draft_no", "ledger_row_count", "voucher_row_count", "ledger_debit", "ledger_credit", "voucher_debit", "voucher_credit", "ledger_accounts", "voucher_accounts", "ledger_vendors", "voucher_vendors", "review_reason", "search_text", ) ROW_COLUMNS = ( "start_year", "end_year", "status_key", "signature", "group_index", "row_index", "fiscal_year", "status_label", "ledger_date", "proof_date", "voucher_no", "draft_no", "ledger_account_name", "voucher_account_name", "ledger_vendor", "voucher_vendor", "ledger_debit", "ledger_credit", "voucher_debit", "voucher_credit", "ledger_desc", "voucher_desc", "review_reason", "matched_case", "ledger_row_key", "voucher_row_key", "match_identity_key", ) def parse_amount(value: Any) -> float: try: return float(str(value or "0").replace(",", "")) except Exception: return 0.0 def amount_key(value: Any) -> str: amount = parse_amount(value) if abs(amount - round(amount)) < 0.0001: return str(int(round(amount))) return f"{amount:.2f}".rstrip("0").rstrip(".") def db_key_from_display(year: int, ledger_date: Any, voucher_no: Any) -> str: voucher = clean(voucher_no) if re.fullmatch(r"\d{8}-\d{5}", voucher): return voucher date_text = clean(ledger_date) full_date_match = re.search(r"((?:19|20)\d{2})[-./](\d{1,2})[-./](\d{1,2})", date_text) date_match = re.search(r"(\d{1,2})[-./](\d{1,2})", date_text) voucher_digits = re.sub(r"\D+", "", voucher) if full_date_match and voucher_digits: return f"{int(full_date_match.group(1)):04d}{int(full_date_match.group(2)):02d}{int(full_date_match.group(3)):02d}-{int(voucher_digits):05d}" if date_match and voucher_digits: return f"{int(year):04d}{int(date_match.group(1)):02d}{int(date_match.group(2)):02d}-{int(voucher_digits):05d}" return "" def display_date_from_db_key(db_key: str) -> str: return f"{db_key[4:6]}-{db_key[6:8]}" if re.fullmatch(r"\d{8}-\d{5}", db_key) else "" def display_voucher_from_db_key(db_key: str) -> str: return db_key.split("-", 1)[1] if "-" in db_key else db_key def erp_voucher_base(value: Any) -> str: text = clean(value) match = re.fullmatch(r"(11-\d{8}-[^-]+-\d+)-\d+", text) if match: return match.group(1) return text def dict_row(row: sqlite3.Row) -> dict[str, Any]: return {key: row[key] for key in row.keys()} def latest_projection_signature(cur: sqlite3.Cursor) -> str: row = cur.execute( """ SELECT payload_json FROM wehago_action_history WHERE action_type = 'auto_recheck_promote' ORDER BY id DESC LIMIT 1 """ ).fetchone() if row: try: payload = json.loads(row[0] or "{}") signature = clean(payload.get("signature")) if signature and "|db-reconciled-v1|" not in signature: exists = cur.execute( """ SELECT 1 FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ? LIMIT 1 """, (YEAR, YEAR, signature), ).fetchone() if exists: return signature except Exception: pass row = cur.execute( """ SELECT signature, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature LIKE ? AND signature NOT LIKE '%|db-reconciled-v1|%' GROUP BY signature ORDER BY max_updated_at DESC LIMIT 1 """, (YEAR, YEAR, f"{QUERY_PROJECTION_VERSION}|%"), ).fetchone() if not row: row = cur.execute( """ SELECT signature, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature NOT LIKE '%|db-reconciled-v1|%' GROUP BY signature ORDER BY max_updated_at DESC LIMIT 1 """, (YEAR, YEAR), ).fetchone() if not row: raise RuntimeError("No current query projection was found.") return clean(row["signature"]) def load_groups(conn: sqlite3.Connection, signature: str) -> dict[str, list[dict[str, Any]]]: group_rows = conn.execute( f""" SELECT * FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ? AND status_key IN ({','.join('?' for _ in ALL_VOUCHER_STATUSES)}) ORDER BY status_key, group_index """, (YEAR, YEAR, signature, *ALL_VOUCHER_STATUSES), ).fetchall() detail_rows = conn.execute( f""" SELECT * FROM wehago_compare_query_rows WHERE start_year = ? AND end_year = ? AND signature = ? AND status_key IN ({','.join('?' for _ in ALL_VOUCHER_STATUSES)}) ORDER BY status_key, group_index, row_index """, (YEAR, YEAR, signature, *ALL_VOUCHER_STATUSES), ).fetchall() rows_by_key: dict[tuple[str, int], list[dict[str, Any]]] = defaultdict(list) for row in detail_rows: item = dict_row(row) rows_by_key[(clean(item.get("status_key")), int(item.get("group_index") or 0))].append(item) groups: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in group_rows: summary = dict_row(row) status_key = clean(summary.get("status_key")) group_index = int(summary.get("group_index") or 0) groups[status_key].append( { "summary": summary, "rows": rows_by_key.get((status_key, group_index), []), "source_group_index": group_index, } ) return groups def load_db_wehago(conn: sqlite3.Connection) -> dict[str, dict[str, Any]]: rows = conn.execute( """ SELECT * FROM wehago_comparison_results WHERE fiscal_year = ? AND status <> 'voucher_only' ORDER BY voucher_no """, (YEAR,), ).fetchall() return {clean(row["voucher_no"]): dict_row(row) for row in rows if clean(row["voucher_no"])} def load_raw_erp_rows_by_draft_base(conn: sqlite3.Connection) -> dict[str, list[dict[str, Any]]]: result: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in conn.execute( """ SELECT * FROM wehago_voucher_rows WHERE fiscal_year BETWEEN ? AND ? AND (COALESCE(draft_no, '') <> '' OR COALESCE(confirmed_no, '') <> '') ORDER BY row_number """, (YEAR - 1, YEAR + 1), ).fetchall(): payload = dict_row(row) keys = { erp_voucher_base(payload.get("draft_no")), erp_voucher_base(payload.get("confirmed_no")), } for key in keys: if key: result[key].append(payload) return result def group_identity(group: dict[str, Any]) -> str: summary = group.get("summary") or {} return db_key_from_display( int(summary.get("fiscal_year") or YEAR), summary.get("ledger_date"), summary.get("voucher_no"), ) def load_manual_offset_excepted_identities(conn: sqlite3.Connection) -> set[str]: conn.execute( """ CREATE TABLE IF NOT EXISTS wehago_manual_offset_excepted ( pair_key TEXT PRIMARY KEY, left_identity TEXT NOT NULL, right_identity TEXT NOT NULL, start_year INTEGER NOT NULL, end_year INTEGER NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ) """ ) identities: set[str] = set() for row in conn.execute( """ SELECT left_identity, right_identity FROM wehago_manual_offset_excepted """ ).fetchall(): identities.update(filter(None, (clean(row["left_identity"]), clean(row["right_identity"])))) return identities def row_wehago_identity(row: dict[str, Any], fallback_group: dict[str, Any] | None = None) -> str: row_key = db_key_from_display( int(row.get("fiscal_year") or YEAR), row.get("ledger_date"), row.get("voucher_no"), ) if row_key: return row_key if fallback_group: return group_identity(fallback_group) return "" def has_erp_value(row: dict[str, Any]) -> bool: return bool(clean(row.get("voucher_account_name"))) and ( abs(parse_amount(row.get("voucher_debit"))) > 0.0001 or abs(parse_amount(row.get("voucher_credit"))) > 0.0001 ) def has_wehago_value(row: dict[str, Any]) -> bool: return bool(clean(row.get("ledger_account_name"))) and ( abs(parse_amount(row.get("ledger_debit"))) > 0.0001 or abs(parse_amount(row.get("ledger_credit"))) > 0.0001 ) def blank_erp_side(row: dict[str, Any], reason: str) -> dict[str, Any]: payload = dict(row) payload["status_label"] = "Unmatched" payload["proof_date"] = "" payload["draft_no"] = "" payload["voucher_account_name"] = "" payload["voucher_vendor"] = "" payload["voucher_debit"] = 0 payload["voucher_credit"] = 0 payload["voucher_desc"] = "" payload["voucher_row_key"] = "" payload["match_identity_key"] = "" payload["review_reason"] = reason return payload def blank_wehago_side(row: dict[str, Any], reason: str) -> dict[str, Any]: payload = dict(row) payload["status_label"] = "ERP Unmatched" payload["ledger_date"] = "" payload["ledger_account_name"] = "" payload["ledger_vendor"] = "" payload["ledger_debit"] = 0 payload["ledger_credit"] = 0 payload["ledger_desc"] = "" payload["ledger_row_key"] = "" payload["match_identity_key"] = "" payload["review_reason"] = reason return payload def ledger_row_identity(row: dict[str, Any]) -> tuple[Any, ...]: row_key = clean(row.get("ledger_row_key")) if row_key: return ("key", row_key) return ( "side", clean(row.get("ledger_account_name")), clean(row.get("ledger_vendor")), amount_key(row.get("ledger_debit")), amount_key(row.get("ledger_credit")), clean(row.get("ledger_desc")), ) def voucher_row_identity(row: dict[str, Any]) -> tuple[Any, ...]: row_key = clean(row.get("voucher_row_key")) if row_key: return ("key", row_key) return ( "side", clean(row.get("draft_no")), clean(row.get("voucher_account_name")), clean(row.get("voucher_vendor")), amount_key(row.get("voucher_debit")), amount_key(row.get("voucher_credit")), clean(row.get("voucher_desc")), ) def direct_row_score(row: dict[str, Any]) -> tuple[int, float, int]: same_amount = int( abs(parse_amount(row.get("ledger_debit")) - parse_amount(row.get("voucher_debit"))) < 0.5 and abs(parse_amount(row.get("ledger_credit")) - parse_amount(row.get("voucher_credit"))) < 0.5 ) amount = max( abs(parse_amount(row.get("ledger_debit"))), abs(parse_amount(row.get("ledger_credit"))), abs(parse_amount(row.get("voucher_debit"))), abs(parse_amount(row.get("voucher_credit"))), ) has_vendor = int(bool(clean(row.get("ledger_vendor"))) and clean(row.get("ledger_vendor")) == clean(row.get("voucher_vendor"))) return same_amount, amount, has_vendor def effective_account_side(row: dict[str, Any], prefix: str) -> str: net_debit = parse_amount(row.get(f"{prefix}_debit")) - parse_amount(row.get(f"{prefix}_credit")) if net_debit > 0.0001: return "debit" if net_debit < -0.0001: return "credit" return "either" def direct_row_matches_account_nature(row: dict[str, Any]) -> bool: if not (has_wehago_value(row) and has_erp_value(row)): return True if not _account_category_pair_allowed( "", row.get("ledger_account_name"), "", row.get("voucher_account_name"), ): return False return _nature_compatible( "", row.get("ledger_account_name"), effective_account_side(row, "ledger"), "", row.get("voucher_account_name"), effective_account_side(row, "voucher"), ) def row_side_amount(row: dict[str, Any], prefix: str) -> float: return max( abs(parse_amount(row.get(f"{prefix}_debit"))), abs(parse_amount(row.get(f"{prefix}_credit"))), ) def raw_erp_entry_amount_side(entry: dict[str, Any], category: str) -> tuple[float, str]: fields = ( ("debit_supply", "debit"), ("credit_supply", "credit"), ) if category in {"asset", "liability"}: fields = ( ("debit_supply", "debit"), ("credit_supply", "credit"), ("debit_tax", "debit"), ("credit_tax", "credit"), ) for field, side in fields: amount = parse_amount(entry.get(field)) if abs(amount) >= 0.5: return abs(amount), side return 0.0, "either" def raw_erp_entry_to_row(entry: dict[str, Any], ledger_row: dict[str, Any], side: str, reason: str) -> dict[str, Any]: amount, _entry_side = raw_erp_entry_amount_side( entry, _classify_account_category(entry.get("account_code"), entry.get("account_name")), ) desc = " ".join(part for part in (clean(entry.get("desc1")), clean(entry.get("desc2"))) if part) row = dict(ledger_row) row.update( { "proof_date": clean(entry.get("proof_date")), "draft_no": clean(entry.get("draft_no")) or clean(entry.get("confirmed_no")), "voucher_account_code": clean(entry.get("account_code")), "voucher_account_name": clean(entry.get("account_name")), "voucher_vendor": clean(entry.get("vendor_name")), "voucher_debit": amount if side == "debit" else 0, "voucher_credit": amount if side == "credit" else 0, "voucher_desc": desc, "status_label": "Matched", "review_reason": reason, } ) row["voucher_row_key"] = build_voucher_row_key(row) row["match_identity_key"] = "|".join( clean(part) for part in ( row.get("fiscal_year"), row.get("voucher_no"), row.get("ledger_row_key"), row.get("draft_no"), row.get("voucher_row_key"), ) if clean(part) ) return row def retarget_to_same_draft_business_account(row: dict[str, Any]) -> dict[str, Any] | None: if not (has_wehago_value(row) and clean(row.get("draft_no"))): return None ledger_category = _classify_account_category("", row.get("ledger_account_name")) if ledger_category not in {"expense", "revenue"}: return None ledger_side = effective_account_side(row, "ledger") ledger_amount = row_side_amount(row, "ledger") if ledger_amount <= 0: return None candidate_bases = { erp_voucher_base(part) for part in re.split(r"[,/]\s*", clean(row.get("draft_no"))) if clean(part) } best: tuple[int, dict[str, Any], str] | None = None for base in candidate_bases: for entry in RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, []): voucher_category = _classify_account_category(entry.get("account_code"), entry.get("account_name")) if voucher_category != ledger_category: continue amount, side = raw_erp_entry_amount_side(entry, voucher_category) if abs(amount - ledger_amount) >= 0.5: continue if not _nature_compatible("", row.get("ledger_account_name"), ledger_side, entry.get("account_code"), entry.get("account_name"), side): continue score = 0 if clean(row.get("ledger_vendor")) and clean(row.get("ledger_vendor")) in clean(entry.get("vendor_name")): score += 10 if clean(entry.get("vendor_name")) and clean(entry.get("vendor_name")) in clean(row.get("ledger_vendor")): score += 10 if clean(row.get("ledger_desc")) and clean(row.get("ledger_desc")) == clean(" ".join(part for part in (entry.get("desc1"), entry.get("desc2")) if clean(part))): score += 20 if best is None or score > best[0]: best = (score, entry, side) if best is None: if ledger_category != "expense": return None grouped_best: tuple[int, dict[str, Any], dict[str, Any], str] | None = None for base in candidate_bases: entries = RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, []) business_entries = [ entry for entry in entries if _classify_account_category(entry.get("account_code"), entry.get("account_name")) == "expense" ] if not business_entries: continue for payable_entry in entries: if _classify_account_family(payable_entry.get("account_code"), payable_entry.get("account_name")) != "payable": continue amount, _payable_side = raw_erp_entry_amount_side(payable_entry, "liability") if abs(amount - ledger_amount) >= 0.5: continue for business_entry in business_entries: business_amount, business_side = raw_erp_entry_amount_side(business_entry, "expense") if business_amount + 0.5 < ledger_amount: continue if not _nature_compatible("", row.get("ledger_account_name"), ledger_side, business_entry.get("account_code"), business_entry.get("account_name"), business_side): continue score = 0 payee = clean(payable_entry.get("vendor_name")) ledger_text = f"{clean(row.get('ledger_vendor'))} {clean(row.get('ledger_desc'))}" if payee and payee in ledger_text: score += 40 if clean(payable_entry.get("desc1")) and clean(payable_entry.get("desc1")) in ledger_text: score += 20 if clean(business_entry.get("account_name")) and "여비교통비" in clean(business_entry.get("account_name")) and "여비교통비" in clean(row.get("ledger_account_name")): score += 20 if grouped_best is None or score > grouped_best[0]: grouped_best = (score, business_entry, payable_entry, business_side) if grouped_best is None: return None business_entry = dict(grouped_best[1]) payable_entry = grouped_best[2] business_entry["debit_supply"] = ledger_amount if grouped_best[3] == "debit" else 0 business_entry["credit_supply"] = ledger_amount if grouped_best[3] == "credit" else 0 business_entry["vendor_name"] = clean(payable_entry.get("vendor_name")) or clean(business_entry.get("vendor_name")) business_entry["desc1"] = clean(payable_entry.get("desc1")) or clean(business_entry.get("desc1")) business_entry["management_item"] = clean(payable_entry.get("management_item")) or clean(business_entry.get("management_item")) return raw_erp_entry_to_row( business_entry, row, grouped_best[3], "PROJECTION_RECONCILE_RETARGET_GROUPED_EXPENSE_PAYABLE_DETAIL", ) return raw_erp_entry_to_row(best[1], row, best[2], "PROJECTION_RECONCILE_RETARGET_SAME_DRAFT_BUSINESS_ACCOUNT") def allowed_erp_drafts_for_group(rows: list[dict[str, Any]]) -> set[str]: erp_rows = [row for row in rows if has_erp_value(row) and clean(row.get("draft_no"))] exception_rows: dict[str, set[tuple[Any, ...]]] = defaultdict(set) for row in rows: family = _classify_account_family("", row.get("ledger_account_name")) category = _classify_account_category("", row.get("ledger_account_name")) bucket = "vat" if _is_vat_family(family) else category if category in {"expense", "asset"} else "" if bucket: exception_rows[bucket].add(ledger_row_identity(row)) capacity = max(1, *(len(row_keys) for row_keys in exception_rows.values())) if exception_rows else 1 drafts: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in erp_rows: drafts[erp_voucher_base(row.get("draft_no"))].append(row) ranked = sorted( drafts, key=lambda draft: ( sum(direct_row_score(row)[0] for row in drafts[draft]), sum(direct_row_score(row)[1] for row in drafts[draft]), sum(direct_row_score(row)[2] for row in drafts[draft]), -len(drafts[draft]), draft, ), reverse=True, ) return set(ranked[: max(1, capacity)]) def clean_group_rows(group: dict[str, Any], status_key: str) -> dict[str, Any]: rows = [dict(row) for row in group.get("rows") or []] if status_key == "voucher_matched": normalized_rows: list[dict[str, Any]] = [] for row in rows: if direct_row_matches_account_nature(row): normalized_rows.append(row) continue retargeted = retarget_to_same_draft_business_account(row) if retargeted is not None: normalized_rows.append(retargeted) else: normalized_rows.append(blank_erp_side(row, "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE")) rows = normalized_rows allowed_drafts = allowed_erp_drafts_for_group(rows) cleaned: list[dict[str, Any]] = [] for row in rows: if clean(row.get("voucher_account_name")) and ( not has_erp_value(row) or (clean(row.get("draft_no")) and erp_voucher_base(row.get("draft_no")) not in allowed_drafts) ): if has_wehago_value(row): cleaned.append(blank_erp_side(row, "PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP")) continue else: cleaned.append(row) direct_drafts = { erp_voucher_base(row.get("draft_no")) for row in cleaned if has_wehago_value(row) and has_erp_value(row) and clean(row.get("draft_no")) } rows = [ row for row in cleaned if has_wehago_value(row) or not has_erp_value(row) or erp_voucher_base(row.get("draft_no")) in direct_drafts ] elif status_key == "voucher_recheck": cleaned = [] for row in rows: if clean(row.get("voucher_account_name")) and not has_erp_value(row): if has_wehago_value(row): cleaned.append(blank_erp_side(row, "PROJECTION_RECONCILE_RECHECK_NO_REAL_ERP_CANDIDATE")) continue cleaned.append(row) rows = cleaned seen_ledger: set[tuple[Any, ...]] = set() seen_voucher: set[tuple[Any, ...]] = set() deduped: list[dict[str, Any]] = [] for row in rows: ledger_key = ledger_row_identity(row) if clean(row.get("ledger_account_name")) else None voucher_key = voucher_row_identity(row) if clean(row.get("voucher_account_name")) else None if ledger_key and ledger_key in seen_ledger: continue if voucher_key and voucher_key in seen_voucher: continue if ledger_key: seen_ledger.add(ledger_key) if voucher_key: seen_voucher.add(voucher_key) if clean(row.get("ledger_account_name")) or clean(row.get("voucher_account_name")): deduped.append(row) group = {"summary": dict(group.get("summary") or {}), "rows": deduped, "source_group_index": group.get("source_group_index")} group["summary"] = rebuild_summary(group, status_key) return group def split_group_by_wehago_voucher(group: dict[str, Any], status_key: str) -> dict[str, dict[str, Any]]: rows_by_identity: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in group.get("rows") or []: identity = row_wehago_identity(row, group) if identity: rows_by_identity[identity].append(dict(row)) if not rows_by_identity: identity = group_identity(group) if identity: rows_by_identity[identity] = [dict(row) for row in group.get("rows") or []] split_groups: dict[str, dict[str, Any]] = {} old_summary = dict(group.get("summary") or {}) for identity, rows in rows_by_identity.items(): if not rows: continue ledger_date = display_date_from_db_key(identity) voucher_no = display_voucher_from_db_key(identity) normalized_rows: list[dict[str, Any]] = [] for row in rows: payload = dict(row) if clean(payload.get("ledger_account_name")): payload["ledger_date"] = clean(payload.get("ledger_date")) or ledger_date payload["voucher_no"] = clean(payload.get("voucher_no")) or voucher_no elif clean(payload.get("voucher_account_name")): payload["ledger_date"] = clean(payload.get("ledger_date")) or ledger_date payload["voucher_no"] = clean(payload.get("voucher_no")) or voucher_no normalized_rows.append(payload) summary = { **old_summary, "fiscal_year": YEAR, "ledger_date": ledger_date, "voucher_no": voucher_no, "draft_no": "", "ledger_accounts": "", "voucher_accounts": "", "ledger_vendors": "", "voucher_vendors": "", "review_reason": clean(old_summary.get("review_reason")), "search_text": "", } split_group = { "summary": summary, "rows": normalized_rows, "source_group_index": group.get("source_group_index"), } split_group["summary"] = rebuild_summary(split_group, status_key) split_groups[identity] = split_group return split_groups def normalized_wehago_status(status_key: str, group: dict[str, Any]) -> str: if status_key == "voucher_matched": if any( has_wehago_value(row) and has_erp_value(row) for row in group.get("rows") or [] ): return "voucher_matched" review_text = " ".join( clean(row.get("review_reason")) for row in group.get("rows") or [] ) if any( token in review_text for token in ( "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE", "PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP", ) ): return "voucher_recheck" return "voucher_unmatched" if status_key != "voucher_recheck": return status_key review_text = " ".join( [ clean((group.get("summary") or {}).get("review_reason")), *(clean(row.get("review_reason")) for row in group.get("rows") or []), ] ) if any( reason in review_text for reason in ( "MATCHED_CANCEL_TARGET_RECHECK", "CANCEL_TARGET_ALREADY_MATCHED_RECHECK", "CANCEL_REISSUE_RETARGET_RECHECK", ) ): return "voucher_recheck" has_real_erp_candidate = any(has_erp_value(row) for row in group.get("rows") or []) return "voucher_recheck" if has_real_erp_candidate else "voucher_unmatched" def mark_excepted(group: dict[str, Any], reason: str) -> dict[str, Any]: summary = dict(group.get("summary") or {}) for field in ("proof_date", "draft_no", "voucher_accounts", "voucher_vendors"): summary[field] = "" for field in ("voucher_row_count", "voucher_debit", "voucher_credit"): summary[field] = 0 rows: list[dict[str, Any]] = [] for row in group.get("rows") or []: if not clean(row.get("ledger_account_name")): continue row_payload = dict(row) for field in ("proof_date", "draft_no", "voucher_account_name", "voucher_vendor", "voucher_desc", "voucher_row_key", "match_identity_key"): row_payload[field] = "" row_payload["voucher_debit"] = 0 row_payload["voucher_credit"] = 0 rows.append(row_payload) payload = {"summary": summary, "rows": rows} payload["summary"]["status_label"] = "Excepted" existing_reason = clean(payload["summary"].get("review_reason")) if reason not in existing_reason: payload["summary"]["review_reason"] = " / ".join(item for item in (existing_reason, reason) if item) for row in payload["rows"]: row["status_label"] = "Excepted" row_reason = clean(row.get("review_reason")) if reason not in row_reason: row["review_reason"] = " / ".join(item for item in (row_reason, reason) if item) payload["summary"] = rebuild_summary(payload, "voucher_excepted") return payload def retag_group(group: dict[str, Any], status_key: str, reason: str = "") -> dict[str, Any]: payload = { "summary": dict(group.get("summary") or {}), "rows": [dict(row) for row in group.get("rows") or []], } label = "Matched" if status_key == "voucher_matched" else "Recheck" if status_key == "voucher_recheck" else "Unmatched" existing_reason = clean(payload["summary"].get("review_reason")) if reason and reason not in existing_reason: payload["summary"]["review_reason"] = " / ".join(item for item in (existing_reason, reason) if item) for row in payload["rows"]: row["status_label"] = label row_reason = clean(row.get("review_reason")) if reason and reason not in row_reason: row["review_reason"] = " / ".join(item for item in (row_reason, reason) if item) payload["summary"] = rebuild_summary(payload, status_key) return payload def group_ledger_net_by_account(group: dict[str, Any]) -> dict[str, float]: result: dict[str, float] = defaultdict(float) for row in group.get("rows") or []: account = clean(row.get("ledger_account_name")).replace(" ", "") if not account or not has_wehago_value(row): continue result[account] += parse_amount(row.get("ledger_debit")) - parse_amount(row.get("ledger_credit")) return {key: value for key, value in result.items() if abs(value) >= 0.5} def group_erp_net_mapped_by_ledger_account(group: dict[str, Any]) -> dict[str, float]: result: dict[str, float] = defaultdict(float) for row in group.get("rows") or []: account = clean(row.get("ledger_account_name")).replace(" ", "") if not account or not has_wehago_value(row) or not has_erp_value(row): continue result[account] += parse_amount(row.get("voucher_debit")) - parse_amount(row.get("voucher_credit")) return {key: value for key, value in result.items() if abs(value) >= 0.5} def group_ledger_vendors(group: dict[str, Any]) -> set[str]: return { clean(row.get("ledger_vendor")).replace(" ", "") for row in group.get("rows") or [] if clean(row.get("ledger_vendor")) } def groups_within_days(left: dict[str, Any], right: dict[str, Any], days: int) -> bool: def as_date(group: dict[str, Any]) -> datetime | None: summary = group.get("summary") or {} year = int(summary.get("fiscal_year") or YEAR) text_value = clean(summary.get("ledger_date")) try: return datetime.strptime(f"{year}-{text_value}", "%Y-%m-%d") except ValueError: return None left_date = as_date(left) right_date = as_date(right) return bool(left_date and right_date and abs((left_date - right_date).days) <= days) def apply_net_adjustment_component_matches(status_groups: dict[str, list[dict[str, Any]]]) -> None: matched_groups = list(status_groups.get("voucher_matched") or []) candidate_statuses = ("voucher_unmatched", "voucher_recheck") candidate_groups = [ (status_key, group) for status_key in candidate_statuses for group in list(status_groups.get(status_key) or []) ] matched_index: dict[tuple[tuple[str, ...], str], list[tuple[dict[str, Any], dict[str, float], dict[str, float]]]] = defaultdict(list) for matched in matched_groups: matched_net = group_ledger_net_by_account(matched) erp_net = group_erp_net_mapped_by_ledger_account(matched) vendors = group_ledger_vendors(matched) if not matched_net or set(erp_net) != set(matched_net) or not vendors: continue account_key = tuple(sorted(matched_net)) for vendor in vendors: matched_index[(account_key, vendor)].append((matched, matched_net, erp_net)) moves: dict[int, tuple[dict[str, Any], str]] = {} for candidate_status, candidate in candidate_groups: candidate_net = group_ledger_net_by_account(candidate) if not candidate_net or not any(value < -0.5 for value in candidate_net.values()): continue candidate_vendors = group_ledger_vendors(candidate) if not candidate_vendors: continue possible: dict[int, tuple[dict[str, Any], dict[str, float], dict[str, float]]] = {} account_key = tuple(sorted(candidate_net)) for vendor in candidate_vendors: for matched, matched_net, erp_net in matched_index.get((account_key, vendor), []): possible[id(matched)] = (matched, matched_net, erp_net) matches: list[dict[str, Any]] = [] for matched, matched_net, erp_net in possible.values(): if not groups_within_days(matched, candidate, 93): continue if all(abs(matched_net[key] + candidate_net[key] - erp_net[key]) < 0.5 for key in matched_net): matches.append(matched) if len(matches) != 1: continue matched_summary = matches[0].get("summary") or {} draft_no = clean(matched_summary.get("draft_no")) reason = "NET_ADJUSTMENT_COMPONENT_MATCH" adjusted = retag_group(candidate, "voucher_matched", reason) if draft_no: adjusted["summary"]["draft_no"] = draft_no adjusted["summary"]["search_text"] = f"{clean(adjusted['summary'].get('search_text'))} {draft_no} {reason}".strip() moves[id(candidate)] = (adjusted, candidate_status) if not moves: return for status_key in candidate_statuses: status_groups[status_key] = [ group for group in list(status_groups.get(status_key) or []) if id(group) not in moves ] status_groups["voucher_matched"] = list(status_groups.get("voucher_matched") or []) + [ adjusted for adjusted, _source_status in moves.values() ] def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None: excepted: list[dict[str, Any]] = [] recheck_from_matched: list[dict[str, Any]] = [] unmatched_from_excepted: list[dict[str, Any]] = [] for group in list(status_groups.get("voucher_excepted") or []): existing_reason = clean((group.get("summary") or {}).get("review_reason")) if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES: excepted.append(mark_excepted(group, "MANUAL_OFFSET_PAIR_EXCEPTED")) continue if "WEHAGO_EXCEPTED_OFFSET" in existing_reason or "WEHAGO_EXCEPTED_CONFIRMED_REVERSAL_PAIR" in existing_reason: excepted.append(mark_excepted(group, "")) continue is_excepted, reason = _is_wehago_excepted_voucher_group(group) if is_excepted: excepted.append(mark_excepted(group, reason)) else: recheck_from_matched.append( retag_group(group, "voucher_recheck", "PROJECTION_RECONCILE_EXCEPTED_RULE_RELAXED_RECHECK") ) retained_by_status: dict[str, list[dict[str, Any]]] = {} for status_key in ("voucher_matched", "voucher_unmatched", "voucher_recheck"): retained: list[dict[str, Any]] = [] for group in list(status_groups.get(status_key) or []): if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES: excepted.append(mark_excepted(group, "MANUAL_OFFSET_PAIR_EXCEPTED")) continue is_excepted, reason = _is_wehago_excepted_voucher_group(group) if is_excepted and status_key == "voucher_matched" and reason != "WEHAGO_EXCEPTED_DEPRECIATION_ACCOUNT": recheck_from_matched.append( retag_group(group, "voucher_recheck", f"PROJECTION_RECONCILE_EXCEPTED_CANDIDATE_RECHECK / {reason}") ) elif is_excepted: excepted.append(mark_excepted(group, reason)) else: retained.append(group) retained_by_status[status_key] = retained retained_by_status["voucher_unmatched"].extend(unmatched_from_excepted) retained_by_status["voucher_recheck"].extend(recheck_from_matched) confirmed = _move_wehago_confirmed_reversal_pairs_to_excepted( { "voucher_matched": retained_by_status["voucher_matched"], "voucher_unmatched": retained_by_status["voucher_unmatched"], "voucher_recheck": retained_by_status["voucher_recheck"], "voucher_excepted": excepted, } ) retained_by_status["voucher_matched"] = list(confirmed.get("voucher_matched") or []) retained_by_status["voucher_unmatched"] = list(confirmed.get("voucher_unmatched") or []) retained_by_status["voucher_recheck"] = list(confirmed.get("voucher_recheck") or []) excepted = list(confirmed.get("voucher_excepted") or []) apply_net_adjustment_component_matches(retained_by_status) moved = _move_wehago_offset_tax_invoice_groups_to_excepted( { "voucher_unmatched": retained_by_status["voucher_unmatched"], "voucher_recheck": retained_by_status["voucher_recheck"], "voucher_excepted": excepted, } ) status_groups["voucher_matched"] = retained_by_status["voucher_matched"] status_groups["voucher_unmatched"] = list(moved.get("voucher_unmatched") or []) status_groups["voucher_recheck"] = list(moved.get("voucher_recheck") or []) status_groups["voucher_excepted"] = [ mark_excepted(group, "") for group in list(moved.get("voucher_excepted") or []) ] def append_unique(values: list[str], value: Any) -> None: text = clean(value) if text and text not in values: values.append(text) def rebuild_summary(group: dict[str, Any], status_key: str) -> dict[str, Any]: old = dict(group.get("summary") or {}) rows = list(group.get("rows") or []) ledger_accounts: list[str] = [] voucher_accounts: list[str] = [] ledger_vendors: list[str] = [] voucher_vendors: list[str] = [] draft_nos: list[str] = [] reasons: list[str] = [] summary = { **old, "ledger_row_count": 0, "voucher_row_count": 0, "ledger_debit": 0.0, "ledger_credit": 0.0, "voucher_debit": 0.0, "voucher_credit": 0.0, } for row in rows: if clean(row.get("ledger_account_name")): summary["ledger_row_count"] += 1 summary["ledger_debit"] += parse_amount(row.get("ledger_debit")) summary["ledger_credit"] += parse_amount(row.get("ledger_credit")) append_unique(ledger_accounts, row.get("ledger_account_name")) append_unique(ledger_vendors, row.get("ledger_vendor")) if clean(row.get("voucher_account_name")): summary["voucher_row_count"] += 1 summary["voucher_debit"] += parse_amount(row.get("voucher_debit")) summary["voucher_credit"] += parse_amount(row.get("voucher_credit")) append_unique(voucher_accounts, row.get("voucher_account_name")) append_unique(voucher_vendors, row.get("voucher_vendor")) append_unique(draft_nos, row.get("draft_no")) append_unique(reasons, row.get("review_reason")) if not clean(summary.get("ledger_date")) and clean(row.get("ledger_date")): summary["ledger_date"] = clean(row.get("ledger_date")) if not clean(summary.get("proof_date")) and clean(row.get("proof_date")): summary["proof_date"] = clean(row.get("proof_date")) summary["status_label"] = "Matched" if status_key in {"voucher_matched", "erp_voucher_matched"} else "Recheck" if status_key == "voucher_recheck" else "Unmatched" summary["ledger_accounts"] = ", ".join(ledger_accounts) summary["voucher_accounts"] = ", ".join(voucher_accounts) summary["ledger_vendors"] = ", ".join(ledger_vendors) summary["voucher_vendors"] = ", ".join(voucher_vendors) summary["draft_no"] = ", ".join(draft_nos) or clean(old.get("draft_no")) summary["review_reason"] = " / ".join(reasons) or clean(old.get("review_reason")) summary["search_text"] = " ".join( clean(part) for part in [ summary.get("voucher_no"), summary.get("draft_no"), summary.get("ledger_accounts"), summary.get("voucher_accounts"), summary.get("ledger_vendors"), summary.get("voucher_vendors"), summary.get("review_reason"), ] if clean(part) ) return summary def group_quality(group: dict[str, Any], status_key: str) -> tuple[int, int, float, int]: priority = {"voucher_excepted": 4, "voucher_matched": 3, "voucher_recheck": 2, "voucher_unmatched": 1}.get(status_key, 0) rows = list(group.get("rows") or []) direct_rows = sum(1 for row in rows if has_wehago_value(row) and has_erp_value(row)) amount = max( parse_amount((group.get("summary") or {}).get("ledger_debit")), parse_amount((group.get("summary") or {}).get("ledger_credit")), parse_amount((group.get("summary") or {}).get("voucher_debit")), parse_amount((group.get("summary") or {}).get("voucher_credit")), ) return priority, direct_rows, amount, -len(rows) def build_missing_db_group(db_row: dict[str, Any], ledger_rows: list[sqlite3.Row]) -> dict[str, Any]: key = clean(db_row.get("voucher_no")) rows: list[dict[str, Any]] = [] for index, row in enumerate(ledger_rows): rows.append( { "fiscal_year": YEAR, "status_label": "Unmatched", "ledger_date": clean(row["ledger_date"]) or display_date_from_db_key(key), "proof_date": "", "voucher_no": display_voucher_from_db_key(key), "draft_no": "", "ledger_account_name": clean(row["account_name"]), "voucher_account_name": "", "ledger_vendor": clean(row["vendor_name"]), "voucher_vendor": "", "ledger_debit": parse_amount(row["debit"]), "ledger_credit": parse_amount(row["credit"]), "voucher_debit": 0, "voucher_credit": 0, "ledger_desc": clean(row["description"]), "voucher_desc": "", "review_reason": "PROJECTION_RECONCILE_DB_ONLY_WEHAGO", "matched_case": "", "ledger_row_key": f"ledger:{row['id']}", "voucher_row_key": "", "match_identity_key": "", } ) if not rows: rows.append( { "fiscal_year": YEAR, "status_label": "Unmatched", "ledger_date": display_date_from_db_key(key), "proof_date": "", "voucher_no": display_voucher_from_db_key(key), "draft_no": "", "ledger_account_name": clean(db_row.get("ledger_accounts")), "voucher_account_name": "", "ledger_vendor": clean(db_row.get("ledger_vendors")), "voucher_vendor": "", "ledger_debit": parse_amount(db_row.get("ledger_debit")), "ledger_credit": parse_amount(db_row.get("ledger_credit")), "voucher_debit": 0, "voucher_credit": 0, "ledger_desc": clean(db_row.get("notes")), "voucher_desc": "", "review_reason": "PROJECTION_RECONCILE_DB_ONLY_WEHAGO", "matched_case": "", "ledger_row_key": "", "voucher_row_key": "", "match_identity_key": "", } ) summary = { "fiscal_year": YEAR, "status_label": "Unmatched", "ledger_date": display_date_from_db_key(key), "proof_date": "", "voucher_no": display_voucher_from_db_key(key), "draft_no": "", "ledger_row_count": 0, "voucher_row_count": 0, "ledger_debit": 0.0, "ledger_credit": 0.0, "voucher_debit": 0.0, "voucher_credit": 0.0, "ledger_accounts": "", "voucher_accounts": "", "ledger_vendors": "", "voucher_vendors": "", "review_reason": "PROJECTION_RECONCILE_DB_ONLY_WEHAGO", "search_text": "", } group = {"summary": summary, "rows": rows, "source_group_index": 0} group["summary"] = rebuild_summary(group, "voucher_unmatched") return group def ledger_db_row_identity(row: sqlite3.Row) -> tuple[Any, ...]: return ( "side", clean(row["account_name"]), clean(row["vendor_name"]), amount_key(row["debit"]), amount_key(row["credit"]), clean(row["description"]), ) def build_unmatched_ledger_row(row: sqlite3.Row, db_key: str, reason: str) -> dict[str, Any]: return { "fiscal_year": YEAR, "status_label": "Unmatched", "ledger_date": clean(row["ledger_date"]) or display_date_from_db_key(db_key), "proof_date": "", "voucher_no": display_voucher_from_db_key(db_key), "draft_no": "", "ledger_account_name": clean(row["account_name"]), "voucher_account_name": "", "ledger_vendor": clean(row["vendor_name"]), "voucher_vendor": "", "ledger_debit": parse_amount(row["debit"]), "ledger_credit": parse_amount(row["credit"]), "voucher_debit": 0, "voucher_credit": 0, "ledger_desc": clean(row["description"]), "voucher_desc": "", "review_reason": reason, "matched_case": "", "ledger_row_key": f"ledger:{row['id']}", "voucher_row_key": "", "match_identity_key": "", } def supplement_group_with_missing_wehago_rows( group: dict[str, Any], status_key: str, db_key: str, ledger_rows: list[sqlite3.Row], ) -> dict[str, Any]: if status_key not in {"voucher_matched", "voucher_recheck"} or not ledger_rows: return group rows = [dict(row) for row in group.get("rows") or []] seen = { ledger_row_identity(row) for row in rows if clean(row.get("ledger_account_name")) } added = False for ledger_row in ledger_rows: identity = ledger_db_row_identity(ledger_row) if identity in seen: continue rows.append(build_unmatched_ledger_row(ledger_row, db_key, "PROJECTION_RECONCILE_PARTIAL_MATCH_WEHAGO_ROW_UNMATCHED")) seen.add(identity) added = True if not added: return group supplemented = { "summary": dict(group.get("summary") or {}), "rows": rows, "source_group_index": group.get("source_group_index"), } supplemented["summary"] = rebuild_summary(supplemented, status_key) return supplemented def insert_group(conn: sqlite3.Connection, signature: str, status_key: str, group_index: int, group: dict[str, Any]) -> None: summary = dict(group["summary"]) summary.update( { "start_year": YEAR, "end_year": YEAR, "status_key": status_key, "signature": signature, "group_index": group_index, } ) conn.execute( f""" INSERT INTO wehago_compare_query_groups ({', '.join(GROUP_COLUMNS)}, created_at, updated_at) VALUES ({', '.join('?' for _ in GROUP_COLUMNS)}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, [summary.get(column, "") for column in GROUP_COLUMNS], ) for row_index, row in enumerate(group.get("rows") or []): row = dict(row) row.update( { "start_year": YEAR, "end_year": YEAR, "status_key": status_key, "signature": signature, "group_index": group_index, "row_index": row_index, } ) conn.execute( f""" INSERT INTO wehago_compare_query_rows ({', '.join(ROW_COLUMNS)}, created_at, updated_at) VALUES ({', '.join('?' for _ in ROW_COLUMNS)}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, [row.get(column, "") for column in ROW_COLUMNS], ) def update_snapshot_row_counts(conn: sqlite3.Connection, counts: dict[str, int]) -> None: row = conn.execute( """ SELECT row_counts_json FROM wehago_snapshot_status WHERE fiscal_year = ? LIMIT 1 """, (YEAR,), ).fetchone() merged: dict[str, int] = {} if row: try: parsed = json.loads(row["row_counts_json"] or "{}") if isinstance(parsed, dict): merged = {str(key): int(value or 0) for key, value in parsed.items()} except Exception: merged = {} for status_key, count in counts.items(): merged[status_key] = int(count or 0) payload = json.dumps(merged, ensure_ascii=False) if row: conn.execute( """ UPDATE wehago_snapshot_status SET state = 'ready', row_counts_json = ?, updated_at = CURRENT_TIMESTAMP, last_built_at = CURRENT_TIMESTAMP, error_message = '' WHERE fiscal_year = ? """, (payload, YEAR), ) else: conn.execute( """ INSERT INTO wehago_snapshot_status ( fiscal_year, snapshot_signature, state, row_counts_json, error_message, created_at, updated_at, last_requested_at, last_built_at ) VALUES (?, ?, 'ready', ?, '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, (YEAR, "", payload), ) def main() -> None: global MANUAL_OFFSET_EXCEPTED_IDENTITIES, RAW_ERP_ROWS_BY_DRAFT_BASE, YEAR parser = argparse.ArgumentParser(description="Reconcile a WEHAGO comparison query projection to DB rows.") parser.add_argument("--year", type=int, default=YEAR) args = parser.parse_args() YEAR = int(args.year) conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row cur = conn.cursor() source_signature = latest_projection_signature(cur) reconciled_prefix = f"{QUERY_PROJECTION_VERSION}|db-reconciled-v1|" target_signature = source_signature if source_signature.startswith(reconciled_prefix) else f"{reconciled_prefix}{source_signature}" RAW_ERP_ROWS_BY_DRAFT_BASE = load_raw_erp_rows_by_draft_base(conn) MANUAL_OFFSET_EXCEPTED_IDENTITIES = load_manual_offset_excepted_identities(conn) db_wehago = load_db_wehago(conn) groups = load_groups(conn, source_signature) ledger_rows_by_key: dict[str, list[sqlite3.Row]] = defaultdict(list) for row in conn.execute( """ SELECT * FROM wehago_ledger_rows WHERE fiscal_year = ? AND COALESCE(compare_voucher_no, '') <> '' ORDER BY compare_voucher_no, row_number """, (YEAR,), ).fetchall(): ledger_rows_by_key[clean(row["compare_voucher_no"])].append(row) selected_by_key: dict[str, tuple[str, dict[str, Any]]] = {} duplicate_counter = 0 removed_non_db = 0 for status_key in WEHAGO_STATUSES: for group in groups.get(status_key, []): for identity, split_group in split_group_by_wehago_voucher(group, status_key).items(): split_group = clean_group_rows(split_group, status_key) if not identity or identity not in db_wehago: removed_non_db += 1 continue final_status_key = normalized_wehago_status(status_key, split_group) if final_status_key != status_key: split_group["summary"] = rebuild_summary(split_group, final_status_key) current = selected_by_key.get(identity) if current is None or group_quality(split_group, final_status_key) > group_quality(current[1], current[0]): if current is not None: duplicate_counter += 1 selected_by_key[identity] = (final_status_key, split_group) else: duplicate_counter += 1 missing_db = sorted(set(db_wehago) - set(selected_by_key)) for key in missing_db: selected_by_key[key] = ( "voucher_unmatched", build_missing_db_group(db_wehago[key], ledger_rows_by_key.get(key, [])), ) status_groups: dict[str, list[dict[str, Any]]] = defaultdict(list) for key in sorted(selected_by_key): status_key, group = selected_by_key[key] group = supplement_group_with_missing_wehago_rows( group, status_key, key, ledger_rows_by_key.get(key, []), ) status_groups[status_key].append(group) move_excepted_groups(status_groups) excepted_wehago_keys = { group_identity(group) for group in status_groups.get("voucher_excepted") or [] if group_identity(group) } allowed_drafts_by_wehago = { key: allowed_erp_drafts_for_group(group.get("rows") or []) for group in status_groups.get("voucher_matched") or [] for key in [group_identity(group)] if key } for status_key in ERP_STATUSES: seen_erp: set[str] = set() for group in groups.get(status_key, []): cleaned = clean_group_rows(group, status_key) if status_key == "erp_voucher_matched": normalized_rows: list[dict[str, Any]] = [] for row in cleaned.get("rows") or []: identity = row_wehago_identity(row, cleaned) draft = erp_voucher_base(row.get("draft_no")) allowed_drafts = allowed_drafts_by_wehago.get(identity) if has_wehago_value(row) and has_erp_value(row) and not direct_row_matches_account_nature(row): normalized_rows.append(blank_wehago_side(row, "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE")) continue if has_wehago_value(row) and identity in excepted_wehago_keys: if has_erp_value(row): normalized_rows.append(blank_wehago_side(row, "PROJECTION_RECONCILE_WEHAGO_EXCEPTED")) continue if has_wehago_value(row) and allowed_drafts is not None and draft not in allowed_drafts: if has_erp_value(row): normalized_rows.append(blank_wehago_side(row, "PROJECTION_RECONCILE_UNMATCHED_WEHAGO_OVER_CAP")) continue normalized_rows.append(row) cleaned["rows"] = normalized_rows final_status_key = ( "erp_voucher_matched" if any(has_wehago_value(row) and has_erp_value(row) for row in normalized_rows) else "erp_voucher_unmatched" ) cleaned["summary"] = rebuild_summary(cleaned, final_status_key) else: final_status_key = status_key summary = cleaned.get("summary") or {} identity = "|".join( clean(part) for part in (summary.get("fiscal_year"), summary.get("proof_date"), erp_voucher_base(summary.get("draft_no")) or summary.get("voucher_no")) if clean(part) ) if not identity or identity in seen_erp: continue seen_erp.add(identity) status_groups[final_status_key].append(cleaned) conn.execute("BEGIN") try: conn.execute( "DELETE FROM wehago_compare_query_rows WHERE start_year = ? AND end_year = ? AND signature = ?", (YEAR, YEAR, target_signature), ) conn.execute( "DELETE FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ?", (YEAR, YEAR, target_signature), ) counts: dict[str, int] = {} for status_key in ALL_VOUCHER_STATUSES: for group_index, group in enumerate(status_groups.get(status_key, []), start=1): insert_group(conn, target_signature, status_key, group_index, group) counts[status_key] = len(status_groups.get(status_key, [])) update_snapshot_row_counts(conn, counts) conn.execute( """ INSERT INTO wehago_action_history (action_type, payload_json, created_at) VALUES ('reconcile_wehago_projection', ?, CURRENT_TIMESTAMP) """, ( json.dumps( { "start_year": YEAR, "end_year": YEAR, "signature": target_signature, "source_signature": source_signature, "db_wehago_count": len(db_wehago), "counts": counts, "removed_duplicate_groups": duplicate_counter, "removed_non_db_groups": removed_non_db, "added_db_only_groups": len(missing_db), "created_at": datetime.now().isoformat(timespec="seconds"), }, ensure_ascii=False, ), ), ) conn.commit() except Exception: conn.rollback() raise print( json.dumps( { "source_signature": source_signature, "target_signature": target_signature, "db_wehago_count": len(db_wehago), "counts": counts, "wehago_sum": sum(counts.get(status, 0) for status in WEHAGO_STATUSES), "removed_duplicate_groups": duplicate_counter, "removed_non_db_groups": removed_non_db, "added_db_only_groups": len(missing_db), }, ensure_ascii=False, ) ) conn.close() if __name__ == "__main__": main()