1497 lines
59 KiB
Python
1497 lines
59 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from runtime_config import DB_PATH
|
|
from wehago_compare import (
|
|
QUERY_PROJECTION_VERSION,
|
|
_account_category_pair_allowed,
|
|
_classify_account_category,
|
|
_classify_account_family,
|
|
_is_wehago_excepted_voucher_group,
|
|
_is_vat_family,
|
|
_move_wehago_confirmed_reversal_pairs_to_excepted,
|
|
_move_wehago_offset_tax_invoice_groups_to_excepted,
|
|
_nature_compatible,
|
|
build_voucher_row_key,
|
|
clean,
|
|
)
|
|
|
|
|
|
YEAR = 2025
|
|
WEHAGO_STATUSES = ("voucher_matched", "voucher_unmatched", "voucher_recheck", "voucher_excepted")
|
|
ERP_STATUSES = ("erp_voucher_matched", "erp_voucher_unmatched")
|
|
ALL_VOUCHER_STATUSES = WEHAGO_STATUSES + ERP_STATUSES
|
|
RAW_ERP_ROWS_BY_DRAFT_BASE: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
MANUAL_OFFSET_EXCEPTED_IDENTITIES: set[str] = set()
|
|
|
|
GROUP_COLUMNS = (
|
|
"start_year",
|
|
"end_year",
|
|
"status_key",
|
|
"signature",
|
|
"group_index",
|
|
"fiscal_year",
|
|
"ledger_date",
|
|
"proof_date",
|
|
"voucher_no",
|
|
"draft_no",
|
|
"ledger_row_count",
|
|
"voucher_row_count",
|
|
"ledger_debit",
|
|
"ledger_credit",
|
|
"voucher_debit",
|
|
"voucher_credit",
|
|
"ledger_accounts",
|
|
"voucher_accounts",
|
|
"ledger_vendors",
|
|
"voucher_vendors",
|
|
"review_reason",
|
|
"search_text",
|
|
)
|
|
|
|
ROW_COLUMNS = (
|
|
"start_year",
|
|
"end_year",
|
|
"status_key",
|
|
"signature",
|
|
"group_index",
|
|
"row_index",
|
|
"fiscal_year",
|
|
"status_label",
|
|
"ledger_date",
|
|
"proof_date",
|
|
"voucher_no",
|
|
"draft_no",
|
|
"ledger_account_name",
|
|
"voucher_account_name",
|
|
"ledger_vendor",
|
|
"voucher_vendor",
|
|
"ledger_debit",
|
|
"ledger_credit",
|
|
"voucher_debit",
|
|
"voucher_credit",
|
|
"ledger_desc",
|
|
"voucher_desc",
|
|
"review_reason",
|
|
"matched_case",
|
|
"ledger_row_key",
|
|
"voucher_row_key",
|
|
"match_identity_key",
|
|
)
|
|
|
|
def parse_amount(value: Any) -> float:
|
|
try:
|
|
return float(str(value or "0").replace(",", ""))
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
def amount_key(value: Any) -> str:
|
|
amount = parse_amount(value)
|
|
if abs(amount - round(amount)) < 0.0001:
|
|
return str(int(round(amount)))
|
|
return f"{amount:.2f}".rstrip("0").rstrip(".")
|
|
|
|
|
|
def db_key_from_display(year: int, ledger_date: Any, voucher_no: Any) -> str:
|
|
voucher = clean(voucher_no)
|
|
if re.fullmatch(r"\d{8}-\d{5}", voucher):
|
|
return voucher
|
|
date_text = clean(ledger_date)
|
|
full_date_match = re.search(r"((?:19|20)\d{2})[-./](\d{1,2})[-./](\d{1,2})", date_text)
|
|
date_match = re.search(r"(\d{1,2})[-./](\d{1,2})", date_text)
|
|
voucher_digits = re.sub(r"\D+", "", voucher)
|
|
if full_date_match and voucher_digits:
|
|
return f"{int(full_date_match.group(1)):04d}{int(full_date_match.group(2)):02d}{int(full_date_match.group(3)):02d}-{int(voucher_digits):05d}"
|
|
if date_match and voucher_digits:
|
|
return f"{int(year):04d}{int(date_match.group(1)):02d}{int(date_match.group(2)):02d}-{int(voucher_digits):05d}"
|
|
return ""
|
|
|
|
|
|
def display_date_from_db_key(db_key: str) -> str:
|
|
return f"{db_key[4:6]}-{db_key[6:8]}" if re.fullmatch(r"\d{8}-\d{5}", db_key) else ""
|
|
|
|
|
|
def display_voucher_from_db_key(db_key: str) -> str:
|
|
return db_key.split("-", 1)[1] if "-" in db_key else db_key
|
|
|
|
|
|
def erp_voucher_base(value: Any) -> str:
|
|
text = clean(value)
|
|
match = re.fullmatch(r"(11-\d{8}-[^-]+-\d+)-\d+", text)
|
|
if match:
|
|
return match.group(1)
|
|
return text
|
|
|
|
|
|
def dict_row(row: sqlite3.Row) -> dict[str, Any]:
|
|
return {key: row[key] for key in row.keys()}
|
|
|
|
|
|
def latest_projection_signature(cur: sqlite3.Cursor) -> str:
|
|
row = cur.execute(
|
|
"""
|
|
SELECT payload_json
|
|
FROM wehago_action_history
|
|
WHERE action_type = 'auto_recheck_promote'
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
"""
|
|
).fetchone()
|
|
if row:
|
|
try:
|
|
payload = json.loads(row[0] or "{}")
|
|
signature = clean(payload.get("signature"))
|
|
if signature and "|db-reconciled-v1|" not in signature:
|
|
exists = cur.execute(
|
|
"""
|
|
SELECT 1
|
|
FROM wehago_compare_query_groups
|
|
WHERE start_year = ? AND end_year = ? AND signature = ?
|
|
LIMIT 1
|
|
""",
|
|
(YEAR, YEAR, signature),
|
|
).fetchone()
|
|
if exists:
|
|
return signature
|
|
except Exception:
|
|
pass
|
|
row = cur.execute(
|
|
"""
|
|
SELECT signature, MAX(updated_at) AS max_updated_at
|
|
FROM wehago_compare_query_groups
|
|
WHERE start_year = ? AND end_year = ? AND signature LIKE ?
|
|
AND signature NOT LIKE '%|db-reconciled-v1|%'
|
|
GROUP BY signature
|
|
ORDER BY max_updated_at DESC
|
|
LIMIT 1
|
|
""",
|
|
(YEAR, YEAR, f"{QUERY_PROJECTION_VERSION}|%"),
|
|
).fetchone()
|
|
if not row:
|
|
row = cur.execute(
|
|
"""
|
|
SELECT signature, MAX(updated_at) AS max_updated_at
|
|
FROM wehago_compare_query_groups
|
|
WHERE start_year = ? AND end_year = ?
|
|
AND signature NOT LIKE '%|db-reconciled-v1|%'
|
|
GROUP BY signature
|
|
ORDER BY max_updated_at DESC
|
|
LIMIT 1
|
|
""",
|
|
(YEAR, YEAR),
|
|
).fetchone()
|
|
if not row:
|
|
raise RuntimeError("No current query projection was found.")
|
|
return clean(row["signature"])
|
|
|
|
|
|
def load_groups(conn: sqlite3.Connection, signature: str) -> dict[str, list[dict[str, Any]]]:
|
|
group_rows = conn.execute(
|
|
f"""
|
|
SELECT *
|
|
FROM wehago_compare_query_groups
|
|
WHERE start_year = ? AND end_year = ? AND signature = ?
|
|
AND status_key IN ({','.join('?' for _ in ALL_VOUCHER_STATUSES)})
|
|
ORDER BY status_key, group_index
|
|
""",
|
|
(YEAR, YEAR, signature, *ALL_VOUCHER_STATUSES),
|
|
).fetchall()
|
|
detail_rows = conn.execute(
|
|
f"""
|
|
SELECT *
|
|
FROM wehago_compare_query_rows
|
|
WHERE start_year = ? AND end_year = ? AND signature = ?
|
|
AND status_key IN ({','.join('?' for _ in ALL_VOUCHER_STATUSES)})
|
|
ORDER BY status_key, group_index, row_index
|
|
""",
|
|
(YEAR, YEAR, signature, *ALL_VOUCHER_STATUSES),
|
|
).fetchall()
|
|
rows_by_key: dict[tuple[str, int], list[dict[str, Any]]] = defaultdict(list)
|
|
for row in detail_rows:
|
|
item = dict_row(row)
|
|
rows_by_key[(clean(item.get("status_key")), int(item.get("group_index") or 0))].append(item)
|
|
groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in group_rows:
|
|
summary = dict_row(row)
|
|
status_key = clean(summary.get("status_key"))
|
|
group_index = int(summary.get("group_index") or 0)
|
|
groups[status_key].append(
|
|
{
|
|
"summary": summary,
|
|
"rows": rows_by_key.get((status_key, group_index), []),
|
|
"source_group_index": group_index,
|
|
}
|
|
)
|
|
return groups
|
|
|
|
|
|
def load_db_wehago(conn: sqlite3.Connection) -> dict[str, dict[str, Any]]:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT *
|
|
FROM wehago_comparison_results
|
|
WHERE fiscal_year = ?
|
|
AND status <> 'voucher_only'
|
|
ORDER BY voucher_no
|
|
""",
|
|
(YEAR,),
|
|
).fetchall()
|
|
return {clean(row["voucher_no"]): dict_row(row) for row in rows if clean(row["voucher_no"])}
|
|
|
|
|
|
def load_raw_erp_rows_by_draft_base(conn: sqlite3.Connection) -> dict[str, list[dict[str, Any]]]:
|
|
result: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT *
|
|
FROM wehago_voucher_rows
|
|
WHERE fiscal_year BETWEEN ? AND ?
|
|
AND (COALESCE(draft_no, '') <> '' OR COALESCE(confirmed_no, '') <> '')
|
|
ORDER BY row_number
|
|
""",
|
|
(YEAR - 1, YEAR + 1),
|
|
).fetchall():
|
|
payload = dict_row(row)
|
|
keys = {
|
|
erp_voucher_base(payload.get("draft_no")),
|
|
erp_voucher_base(payload.get("confirmed_no")),
|
|
}
|
|
for key in keys:
|
|
if key:
|
|
result[key].append(payload)
|
|
return result
|
|
|
|
|
|
def group_identity(group: dict[str, Any]) -> str:
|
|
summary = group.get("summary") or {}
|
|
return db_key_from_display(
|
|
int(summary.get("fiscal_year") or YEAR),
|
|
summary.get("ledger_date"),
|
|
summary.get("voucher_no"),
|
|
)
|
|
|
|
|
|
def load_manual_offset_excepted_identities(conn: sqlite3.Connection) -> set[str]:
|
|
conn.execute(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS wehago_manual_offset_excepted (
|
|
pair_key TEXT PRIMARY KEY,
|
|
left_identity TEXT NOT NULL,
|
|
right_identity TEXT NOT NULL,
|
|
start_year INTEGER NOT NULL,
|
|
end_year INTEGER NOT NULL,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""
|
|
)
|
|
identities: set[str] = set()
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT left_identity, right_identity
|
|
FROM wehago_manual_offset_excepted
|
|
"""
|
|
).fetchall():
|
|
identities.update(filter(None, (clean(row["left_identity"]), clean(row["right_identity"]))))
|
|
return identities
|
|
|
|
|
|
def row_wehago_identity(row: dict[str, Any], fallback_group: dict[str, Any] | None = None) -> str:
|
|
row_key = db_key_from_display(
|
|
int(row.get("fiscal_year") or YEAR),
|
|
row.get("ledger_date"),
|
|
row.get("voucher_no"),
|
|
)
|
|
if row_key:
|
|
return row_key
|
|
if fallback_group:
|
|
return group_identity(fallback_group)
|
|
return ""
|
|
|
|
|
|
def has_erp_value(row: dict[str, Any]) -> bool:
|
|
return bool(clean(row.get("voucher_account_name"))) and (
|
|
abs(parse_amount(row.get("voucher_debit"))) > 0.0001
|
|
or abs(parse_amount(row.get("voucher_credit"))) > 0.0001
|
|
)
|
|
|
|
|
|
def has_wehago_value(row: dict[str, Any]) -> bool:
|
|
return bool(clean(row.get("ledger_account_name"))) and (
|
|
abs(parse_amount(row.get("ledger_debit"))) > 0.0001
|
|
or abs(parse_amount(row.get("ledger_credit"))) > 0.0001
|
|
)
|
|
|
|
|
|
def blank_erp_side(row: dict[str, Any], reason: str) -> dict[str, Any]:
|
|
payload = dict(row)
|
|
payload["status_label"] = "Unmatched"
|
|
payload["proof_date"] = ""
|
|
payload["draft_no"] = ""
|
|
payload["voucher_account_name"] = ""
|
|
payload["voucher_vendor"] = ""
|
|
payload["voucher_debit"] = 0
|
|
payload["voucher_credit"] = 0
|
|
payload["voucher_desc"] = ""
|
|
payload["voucher_row_key"] = ""
|
|
payload["match_identity_key"] = ""
|
|
payload["review_reason"] = reason
|
|
return payload
|
|
|
|
|
|
def blank_wehago_side(row: dict[str, Any], reason: str) -> dict[str, Any]:
|
|
payload = dict(row)
|
|
payload["status_label"] = "ERP Unmatched"
|
|
payload["ledger_date"] = ""
|
|
payload["ledger_account_name"] = ""
|
|
payload["ledger_vendor"] = ""
|
|
payload["ledger_debit"] = 0
|
|
payload["ledger_credit"] = 0
|
|
payload["ledger_desc"] = ""
|
|
payload["ledger_row_key"] = ""
|
|
payload["match_identity_key"] = ""
|
|
payload["review_reason"] = reason
|
|
return payload
|
|
|
|
|
|
def ledger_row_identity(row: dict[str, Any]) -> tuple[Any, ...]:
|
|
row_key = clean(row.get("ledger_row_key"))
|
|
if row_key:
|
|
return ("key", row_key)
|
|
return (
|
|
"side",
|
|
clean(row.get("ledger_account_name")),
|
|
clean(row.get("ledger_vendor")),
|
|
amount_key(row.get("ledger_debit")),
|
|
amount_key(row.get("ledger_credit")),
|
|
clean(row.get("ledger_desc")),
|
|
)
|
|
|
|
|
|
def voucher_row_identity(row: dict[str, Any]) -> tuple[Any, ...]:
|
|
row_key = clean(row.get("voucher_row_key"))
|
|
if row_key:
|
|
return ("key", row_key)
|
|
return (
|
|
"side",
|
|
clean(row.get("draft_no")),
|
|
clean(row.get("voucher_account_name")),
|
|
clean(row.get("voucher_vendor")),
|
|
amount_key(row.get("voucher_debit")),
|
|
amount_key(row.get("voucher_credit")),
|
|
clean(row.get("voucher_desc")),
|
|
)
|
|
|
|
|
|
def direct_row_score(row: dict[str, Any]) -> tuple[int, float, int]:
|
|
same_amount = int(
|
|
abs(parse_amount(row.get("ledger_debit")) - parse_amount(row.get("voucher_debit"))) < 0.5
|
|
and abs(parse_amount(row.get("ledger_credit")) - parse_amount(row.get("voucher_credit"))) < 0.5
|
|
)
|
|
amount = max(
|
|
abs(parse_amount(row.get("ledger_debit"))),
|
|
abs(parse_amount(row.get("ledger_credit"))),
|
|
abs(parse_amount(row.get("voucher_debit"))),
|
|
abs(parse_amount(row.get("voucher_credit"))),
|
|
)
|
|
has_vendor = int(bool(clean(row.get("ledger_vendor"))) and clean(row.get("ledger_vendor")) == clean(row.get("voucher_vendor")))
|
|
return same_amount, amount, has_vendor
|
|
|
|
|
|
def effective_account_side(row: dict[str, Any], prefix: str) -> str:
|
|
net_debit = parse_amount(row.get(f"{prefix}_debit")) - parse_amount(row.get(f"{prefix}_credit"))
|
|
if net_debit > 0.0001:
|
|
return "debit"
|
|
if net_debit < -0.0001:
|
|
return "credit"
|
|
return "either"
|
|
|
|
|
|
def direct_row_matches_account_nature(row: dict[str, Any]) -> bool:
|
|
if not (has_wehago_value(row) and has_erp_value(row)):
|
|
return True
|
|
if not _account_category_pair_allowed(
|
|
"",
|
|
row.get("ledger_account_name"),
|
|
"",
|
|
row.get("voucher_account_name"),
|
|
):
|
|
return False
|
|
return _nature_compatible(
|
|
"",
|
|
row.get("ledger_account_name"),
|
|
effective_account_side(row, "ledger"),
|
|
"",
|
|
row.get("voucher_account_name"),
|
|
effective_account_side(row, "voucher"),
|
|
)
|
|
|
|
|
|
def row_side_amount(row: dict[str, Any], prefix: str) -> float:
|
|
return max(
|
|
abs(parse_amount(row.get(f"{prefix}_debit"))),
|
|
abs(parse_amount(row.get(f"{prefix}_credit"))),
|
|
)
|
|
|
|
|
|
def raw_erp_entry_amount_side(entry: dict[str, Any], category: str) -> tuple[float, str]:
|
|
fields = (
|
|
("debit_supply", "debit"),
|
|
("credit_supply", "credit"),
|
|
)
|
|
if category in {"asset", "liability"}:
|
|
fields = (
|
|
("debit_supply", "debit"),
|
|
("credit_supply", "credit"),
|
|
("debit_tax", "debit"),
|
|
("credit_tax", "credit"),
|
|
)
|
|
for field, side in fields:
|
|
amount = parse_amount(entry.get(field))
|
|
if abs(amount) >= 0.5:
|
|
return abs(amount), side
|
|
return 0.0, "either"
|
|
|
|
|
|
def raw_erp_entry_to_row(entry: dict[str, Any], ledger_row: dict[str, Any], side: str, reason: str) -> dict[str, Any]:
|
|
amount, _entry_side = raw_erp_entry_amount_side(
|
|
entry,
|
|
_classify_account_category(entry.get("account_code"), entry.get("account_name")),
|
|
)
|
|
desc = " ".join(part for part in (clean(entry.get("desc1")), clean(entry.get("desc2"))) if part)
|
|
row = dict(ledger_row)
|
|
row.update(
|
|
{
|
|
"proof_date": clean(entry.get("proof_date")),
|
|
"draft_no": clean(entry.get("draft_no")) or clean(entry.get("confirmed_no")),
|
|
"voucher_account_code": clean(entry.get("account_code")),
|
|
"voucher_account_name": clean(entry.get("account_name")),
|
|
"voucher_vendor": clean(entry.get("vendor_name")),
|
|
"voucher_debit": amount if side == "debit" else 0,
|
|
"voucher_credit": amount if side == "credit" else 0,
|
|
"voucher_desc": desc,
|
|
"status_label": "Matched",
|
|
"review_reason": reason,
|
|
}
|
|
)
|
|
row["voucher_row_key"] = build_voucher_row_key(row)
|
|
row["match_identity_key"] = "|".join(
|
|
clean(part)
|
|
for part in (
|
|
row.get("fiscal_year"),
|
|
row.get("voucher_no"),
|
|
row.get("ledger_row_key"),
|
|
row.get("draft_no"),
|
|
row.get("voucher_row_key"),
|
|
)
|
|
if clean(part)
|
|
)
|
|
return row
|
|
|
|
|
|
def retarget_to_same_draft_business_account(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
if not (has_wehago_value(row) and clean(row.get("draft_no"))):
|
|
return None
|
|
ledger_category = _classify_account_category("", row.get("ledger_account_name"))
|
|
if ledger_category not in {"expense", "revenue"}:
|
|
return None
|
|
ledger_side = effective_account_side(row, "ledger")
|
|
ledger_amount = row_side_amount(row, "ledger")
|
|
if ledger_amount <= 0:
|
|
return None
|
|
candidate_bases = {
|
|
erp_voucher_base(part)
|
|
for part in re.split(r"[,/]\s*", clean(row.get("draft_no")))
|
|
if clean(part)
|
|
}
|
|
best: tuple[int, dict[str, Any], str] | None = None
|
|
for base in candidate_bases:
|
|
for entry in RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, []):
|
|
voucher_category = _classify_account_category(entry.get("account_code"), entry.get("account_name"))
|
|
if voucher_category != ledger_category:
|
|
continue
|
|
amount, side = raw_erp_entry_amount_side(entry, voucher_category)
|
|
if abs(amount - ledger_amount) >= 0.5:
|
|
continue
|
|
if not _nature_compatible("", row.get("ledger_account_name"), ledger_side, entry.get("account_code"), entry.get("account_name"), side):
|
|
continue
|
|
score = 0
|
|
if clean(row.get("ledger_vendor")) and clean(row.get("ledger_vendor")) in clean(entry.get("vendor_name")):
|
|
score += 10
|
|
if clean(entry.get("vendor_name")) and clean(entry.get("vendor_name")) in clean(row.get("ledger_vendor")):
|
|
score += 10
|
|
if clean(row.get("ledger_desc")) and clean(row.get("ledger_desc")) == clean(" ".join(part for part in (entry.get("desc1"), entry.get("desc2")) if clean(part))):
|
|
score += 20
|
|
if best is None or score > best[0]:
|
|
best = (score, entry, side)
|
|
if best is None:
|
|
if ledger_category != "expense":
|
|
return None
|
|
grouped_best: tuple[int, dict[str, Any], dict[str, Any], str] | None = None
|
|
for base in candidate_bases:
|
|
entries = RAW_ERP_ROWS_BY_DRAFT_BASE.get(base, [])
|
|
business_entries = [
|
|
entry
|
|
for entry in entries
|
|
if _classify_account_category(entry.get("account_code"), entry.get("account_name")) == "expense"
|
|
]
|
|
if not business_entries:
|
|
continue
|
|
for payable_entry in entries:
|
|
if _classify_account_family(payable_entry.get("account_code"), payable_entry.get("account_name")) != "payable":
|
|
continue
|
|
amount, _payable_side = raw_erp_entry_amount_side(payable_entry, "liability")
|
|
if abs(amount - ledger_amount) >= 0.5:
|
|
continue
|
|
for business_entry in business_entries:
|
|
business_amount, business_side = raw_erp_entry_amount_side(business_entry, "expense")
|
|
if business_amount + 0.5 < ledger_amount:
|
|
continue
|
|
if not _nature_compatible("", row.get("ledger_account_name"), ledger_side, business_entry.get("account_code"), business_entry.get("account_name"), business_side):
|
|
continue
|
|
score = 0
|
|
payee = clean(payable_entry.get("vendor_name"))
|
|
ledger_text = f"{clean(row.get('ledger_vendor'))} {clean(row.get('ledger_desc'))}"
|
|
if payee and payee in ledger_text:
|
|
score += 40
|
|
if clean(payable_entry.get("desc1")) and clean(payable_entry.get("desc1")) in ledger_text:
|
|
score += 20
|
|
if clean(business_entry.get("account_name")) and "여비교통비" in clean(business_entry.get("account_name")) and "여비교통비" in clean(row.get("ledger_account_name")):
|
|
score += 20
|
|
if grouped_best is None or score > grouped_best[0]:
|
|
grouped_best = (score, business_entry, payable_entry, business_side)
|
|
if grouped_best is None:
|
|
return None
|
|
business_entry = dict(grouped_best[1])
|
|
payable_entry = grouped_best[2]
|
|
business_entry["debit_supply"] = ledger_amount if grouped_best[3] == "debit" else 0
|
|
business_entry["credit_supply"] = ledger_amount if grouped_best[3] == "credit" else 0
|
|
business_entry["vendor_name"] = clean(payable_entry.get("vendor_name")) or clean(business_entry.get("vendor_name"))
|
|
business_entry["desc1"] = clean(payable_entry.get("desc1")) or clean(business_entry.get("desc1"))
|
|
business_entry["management_item"] = clean(payable_entry.get("management_item")) or clean(business_entry.get("management_item"))
|
|
return raw_erp_entry_to_row(
|
|
business_entry,
|
|
row,
|
|
grouped_best[3],
|
|
"PROJECTION_RECONCILE_RETARGET_GROUPED_EXPENSE_PAYABLE_DETAIL",
|
|
)
|
|
return raw_erp_entry_to_row(best[1], row, best[2], "PROJECTION_RECONCILE_RETARGET_SAME_DRAFT_BUSINESS_ACCOUNT")
|
|
|
|
|
|
def allowed_erp_drafts_for_group(rows: list[dict[str, Any]]) -> set[str]:
|
|
erp_rows = [row for row in rows if has_erp_value(row) and clean(row.get("draft_no"))]
|
|
exception_rows: dict[str, set[tuple[Any, ...]]] = defaultdict(set)
|
|
for row in rows:
|
|
family = _classify_account_family("", row.get("ledger_account_name"))
|
|
category = _classify_account_category("", row.get("ledger_account_name"))
|
|
bucket = "vat" if _is_vat_family(family) else category if category in {"expense", "asset"} else ""
|
|
if bucket:
|
|
exception_rows[bucket].add(ledger_row_identity(row))
|
|
capacity = max(1, *(len(row_keys) for row_keys in exception_rows.values())) if exception_rows else 1
|
|
drafts: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in erp_rows:
|
|
drafts[erp_voucher_base(row.get("draft_no"))].append(row)
|
|
ranked = sorted(
|
|
drafts,
|
|
key=lambda draft: (
|
|
sum(direct_row_score(row)[0] for row in drafts[draft]),
|
|
sum(direct_row_score(row)[1] for row in drafts[draft]),
|
|
sum(direct_row_score(row)[2] for row in drafts[draft]),
|
|
-len(drafts[draft]),
|
|
draft,
|
|
),
|
|
reverse=True,
|
|
)
|
|
return set(ranked[: max(1, capacity)])
|
|
|
|
|
|
def clean_group_rows(group: dict[str, Any], status_key: str) -> dict[str, Any]:
|
|
rows = [dict(row) for row in group.get("rows") or []]
|
|
if status_key == "voucher_matched":
|
|
normalized_rows: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
if direct_row_matches_account_nature(row):
|
|
normalized_rows.append(row)
|
|
continue
|
|
retargeted = retarget_to_same_draft_business_account(row)
|
|
if retargeted is not None:
|
|
normalized_rows.append(retargeted)
|
|
else:
|
|
normalized_rows.append(blank_erp_side(row, "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE"))
|
|
rows = normalized_rows
|
|
allowed_drafts = allowed_erp_drafts_for_group(rows)
|
|
cleaned: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
if clean(row.get("voucher_account_name")) and (
|
|
not has_erp_value(row)
|
|
or (clean(row.get("draft_no")) and erp_voucher_base(row.get("draft_no")) not in allowed_drafts)
|
|
):
|
|
if has_wehago_value(row):
|
|
cleaned.append(blank_erp_side(row, "PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP"))
|
|
continue
|
|
else:
|
|
cleaned.append(row)
|
|
direct_drafts = {
|
|
erp_voucher_base(row.get("draft_no"))
|
|
for row in cleaned
|
|
if has_wehago_value(row) and has_erp_value(row) and clean(row.get("draft_no"))
|
|
}
|
|
rows = [
|
|
row for row in cleaned
|
|
if has_wehago_value(row)
|
|
or not has_erp_value(row)
|
|
or erp_voucher_base(row.get("draft_no")) in direct_drafts
|
|
]
|
|
elif status_key == "voucher_recheck":
|
|
cleaned = []
|
|
for row in rows:
|
|
if clean(row.get("voucher_account_name")) and not has_erp_value(row):
|
|
if has_wehago_value(row):
|
|
cleaned.append(blank_erp_side(row, "PROJECTION_RECONCILE_RECHECK_NO_REAL_ERP_CANDIDATE"))
|
|
continue
|
|
cleaned.append(row)
|
|
rows = cleaned
|
|
|
|
seen_ledger: set[tuple[Any, ...]] = set()
|
|
seen_voucher: set[tuple[Any, ...]] = set()
|
|
deduped: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
ledger_key = ledger_row_identity(row) if clean(row.get("ledger_account_name")) else None
|
|
voucher_key = voucher_row_identity(row) if clean(row.get("voucher_account_name")) else None
|
|
if ledger_key and ledger_key in seen_ledger:
|
|
continue
|
|
if voucher_key and voucher_key in seen_voucher:
|
|
continue
|
|
if ledger_key:
|
|
seen_ledger.add(ledger_key)
|
|
if voucher_key:
|
|
seen_voucher.add(voucher_key)
|
|
if clean(row.get("ledger_account_name")) or clean(row.get("voucher_account_name")):
|
|
deduped.append(row)
|
|
group = {"summary": dict(group.get("summary") or {}), "rows": deduped, "source_group_index": group.get("source_group_index")}
|
|
group["summary"] = rebuild_summary(group, status_key)
|
|
return group
|
|
|
|
|
|
def split_group_by_wehago_voucher(group: dict[str, Any], status_key: str) -> dict[str, dict[str, Any]]:
|
|
rows_by_identity: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in group.get("rows") or []:
|
|
identity = row_wehago_identity(row, group)
|
|
if identity:
|
|
rows_by_identity[identity].append(dict(row))
|
|
if not rows_by_identity:
|
|
identity = group_identity(group)
|
|
if identity:
|
|
rows_by_identity[identity] = [dict(row) for row in group.get("rows") or []]
|
|
|
|
split_groups: dict[str, dict[str, Any]] = {}
|
|
old_summary = dict(group.get("summary") or {})
|
|
for identity, rows in rows_by_identity.items():
|
|
if not rows:
|
|
continue
|
|
ledger_date = display_date_from_db_key(identity)
|
|
voucher_no = display_voucher_from_db_key(identity)
|
|
normalized_rows: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
payload = dict(row)
|
|
if clean(payload.get("ledger_account_name")):
|
|
payload["ledger_date"] = clean(payload.get("ledger_date")) or ledger_date
|
|
payload["voucher_no"] = clean(payload.get("voucher_no")) or voucher_no
|
|
elif clean(payload.get("voucher_account_name")):
|
|
payload["ledger_date"] = clean(payload.get("ledger_date")) or ledger_date
|
|
payload["voucher_no"] = clean(payload.get("voucher_no")) or voucher_no
|
|
normalized_rows.append(payload)
|
|
summary = {
|
|
**old_summary,
|
|
"fiscal_year": YEAR,
|
|
"ledger_date": ledger_date,
|
|
"voucher_no": voucher_no,
|
|
"draft_no": "",
|
|
"ledger_accounts": "",
|
|
"voucher_accounts": "",
|
|
"ledger_vendors": "",
|
|
"voucher_vendors": "",
|
|
"review_reason": clean(old_summary.get("review_reason")),
|
|
"search_text": "",
|
|
}
|
|
split_group = {
|
|
"summary": summary,
|
|
"rows": normalized_rows,
|
|
"source_group_index": group.get("source_group_index"),
|
|
}
|
|
split_group["summary"] = rebuild_summary(split_group, status_key)
|
|
split_groups[identity] = split_group
|
|
return split_groups
|
|
|
|
|
|
def normalized_wehago_status(status_key: str, group: dict[str, Any]) -> str:
|
|
if status_key == "voucher_matched":
|
|
if any(
|
|
has_wehago_value(row) and has_erp_value(row)
|
|
for row in group.get("rows") or []
|
|
):
|
|
return "voucher_matched"
|
|
review_text = " ".join(
|
|
clean(row.get("review_reason"))
|
|
for row in group.get("rows") or []
|
|
)
|
|
if any(
|
|
token in review_text
|
|
for token in (
|
|
"PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE",
|
|
"PROJECTION_RECONCILE_UNMATCHED_ERP_OVER_CAP",
|
|
)
|
|
):
|
|
return "voucher_recheck"
|
|
return "voucher_unmatched"
|
|
if status_key != "voucher_recheck":
|
|
return status_key
|
|
review_text = " ".join(
|
|
[
|
|
clean((group.get("summary") or {}).get("review_reason")),
|
|
*(clean(row.get("review_reason")) for row in group.get("rows") or []),
|
|
]
|
|
)
|
|
if any(
|
|
reason in review_text
|
|
for reason in (
|
|
"MATCHED_CANCEL_TARGET_RECHECK",
|
|
"CANCEL_TARGET_ALREADY_MATCHED_RECHECK",
|
|
"CANCEL_REISSUE_RETARGET_RECHECK",
|
|
)
|
|
):
|
|
return "voucher_recheck"
|
|
has_real_erp_candidate = any(has_erp_value(row) for row in group.get("rows") or [])
|
|
return "voucher_recheck" if has_real_erp_candidate else "voucher_unmatched"
|
|
|
|
|
|
def mark_excepted(group: dict[str, Any], reason: str) -> dict[str, Any]:
|
|
summary = dict(group.get("summary") or {})
|
|
for field in ("proof_date", "draft_no", "voucher_accounts", "voucher_vendors"):
|
|
summary[field] = ""
|
|
for field in ("voucher_row_count", "voucher_debit", "voucher_credit"):
|
|
summary[field] = 0
|
|
rows: list[dict[str, Any]] = []
|
|
for row in group.get("rows") or []:
|
|
if not clean(row.get("ledger_account_name")):
|
|
continue
|
|
row_payload = dict(row)
|
|
for field in ("proof_date", "draft_no", "voucher_account_name", "voucher_vendor", "voucher_desc", "voucher_row_key", "match_identity_key"):
|
|
row_payload[field] = ""
|
|
row_payload["voucher_debit"] = 0
|
|
row_payload["voucher_credit"] = 0
|
|
rows.append(row_payload)
|
|
payload = {"summary": summary, "rows": rows}
|
|
payload["summary"]["status_label"] = "Excepted"
|
|
existing_reason = clean(payload["summary"].get("review_reason"))
|
|
if reason not in existing_reason:
|
|
payload["summary"]["review_reason"] = " / ".join(item for item in (existing_reason, reason) if item)
|
|
for row in payload["rows"]:
|
|
row["status_label"] = "Excepted"
|
|
row_reason = clean(row.get("review_reason"))
|
|
if reason not in row_reason:
|
|
row["review_reason"] = " / ".join(item for item in (row_reason, reason) if item)
|
|
payload["summary"] = rebuild_summary(payload, "voucher_excepted")
|
|
return payload
|
|
|
|
|
|
def retag_group(group: dict[str, Any], status_key: str, reason: str = "") -> dict[str, Any]:
|
|
payload = {
|
|
"summary": dict(group.get("summary") or {}),
|
|
"rows": [dict(row) for row in group.get("rows") or []],
|
|
}
|
|
label = "Matched" if status_key == "voucher_matched" else "Recheck" if status_key == "voucher_recheck" else "Unmatched"
|
|
existing_reason = clean(payload["summary"].get("review_reason"))
|
|
if reason and reason not in existing_reason:
|
|
payload["summary"]["review_reason"] = " / ".join(item for item in (existing_reason, reason) if item)
|
|
for row in payload["rows"]:
|
|
row["status_label"] = label
|
|
row_reason = clean(row.get("review_reason"))
|
|
if reason and reason not in row_reason:
|
|
row["review_reason"] = " / ".join(item for item in (row_reason, reason) if item)
|
|
payload["summary"] = rebuild_summary(payload, status_key)
|
|
return payload
|
|
|
|
|
|
def group_ledger_net_by_account(group: dict[str, Any]) -> dict[str, float]:
|
|
result: dict[str, float] = defaultdict(float)
|
|
for row in group.get("rows") or []:
|
|
account = clean(row.get("ledger_account_name")).replace(" ", "")
|
|
if not account or not has_wehago_value(row):
|
|
continue
|
|
result[account] += parse_amount(row.get("ledger_debit")) - parse_amount(row.get("ledger_credit"))
|
|
return {key: value for key, value in result.items() if abs(value) >= 0.5}
|
|
|
|
|
|
def group_erp_net_mapped_by_ledger_account(group: dict[str, Any]) -> dict[str, float]:
|
|
result: dict[str, float] = defaultdict(float)
|
|
for row in group.get("rows") or []:
|
|
account = clean(row.get("ledger_account_name")).replace(" ", "")
|
|
if not account or not has_wehago_value(row) or not has_erp_value(row):
|
|
continue
|
|
result[account] += parse_amount(row.get("voucher_debit")) - parse_amount(row.get("voucher_credit"))
|
|
return {key: value for key, value in result.items() if abs(value) >= 0.5}
|
|
|
|
|
|
def group_ledger_vendors(group: dict[str, Any]) -> set[str]:
|
|
return {
|
|
clean(row.get("ledger_vendor")).replace(" ", "")
|
|
for row in group.get("rows") or []
|
|
if clean(row.get("ledger_vendor"))
|
|
}
|
|
|
|
|
|
def groups_within_days(left: dict[str, Any], right: dict[str, Any], days: int) -> bool:
|
|
def as_date(group: dict[str, Any]) -> datetime | None:
|
|
summary = group.get("summary") or {}
|
|
year = int(summary.get("fiscal_year") or YEAR)
|
|
text_value = clean(summary.get("ledger_date"))
|
|
try:
|
|
return datetime.strptime(f"{year}-{text_value}", "%Y-%m-%d")
|
|
except ValueError:
|
|
return None
|
|
|
|
left_date = as_date(left)
|
|
right_date = as_date(right)
|
|
return bool(left_date and right_date and abs((left_date - right_date).days) <= days)
|
|
|
|
|
|
def apply_net_adjustment_component_matches(status_groups: dict[str, list[dict[str, Any]]]) -> None:
|
|
matched_groups = list(status_groups.get("voucher_matched") or [])
|
|
candidate_statuses = ("voucher_unmatched", "voucher_recheck")
|
|
candidate_groups = [
|
|
(status_key, group)
|
|
for status_key in candidate_statuses
|
|
for group in list(status_groups.get(status_key) or [])
|
|
]
|
|
matched_index: dict[tuple[tuple[str, ...], str], list[tuple[dict[str, Any], dict[str, float], dict[str, float]]]] = defaultdict(list)
|
|
for matched in matched_groups:
|
|
matched_net = group_ledger_net_by_account(matched)
|
|
erp_net = group_erp_net_mapped_by_ledger_account(matched)
|
|
vendors = group_ledger_vendors(matched)
|
|
if not matched_net or set(erp_net) != set(matched_net) or not vendors:
|
|
continue
|
|
account_key = tuple(sorted(matched_net))
|
|
for vendor in vendors:
|
|
matched_index[(account_key, vendor)].append((matched, matched_net, erp_net))
|
|
moves: dict[int, tuple[dict[str, Any], str]] = {}
|
|
for candidate_status, candidate in candidate_groups:
|
|
candidate_net = group_ledger_net_by_account(candidate)
|
|
if not candidate_net or not any(value < -0.5 for value in candidate_net.values()):
|
|
continue
|
|
candidate_vendors = group_ledger_vendors(candidate)
|
|
if not candidate_vendors:
|
|
continue
|
|
possible: dict[int, tuple[dict[str, Any], dict[str, float], dict[str, float]]] = {}
|
|
account_key = tuple(sorted(candidate_net))
|
|
for vendor in candidate_vendors:
|
|
for matched, matched_net, erp_net in matched_index.get((account_key, vendor), []):
|
|
possible[id(matched)] = (matched, matched_net, erp_net)
|
|
matches: list[dict[str, Any]] = []
|
|
for matched, matched_net, erp_net in possible.values():
|
|
if not groups_within_days(matched, candidate, 93):
|
|
continue
|
|
if all(abs(matched_net[key] + candidate_net[key] - erp_net[key]) < 0.5 for key in matched_net):
|
|
matches.append(matched)
|
|
if len(matches) != 1:
|
|
continue
|
|
matched_summary = matches[0].get("summary") or {}
|
|
draft_no = clean(matched_summary.get("draft_no"))
|
|
reason = "NET_ADJUSTMENT_COMPONENT_MATCH"
|
|
adjusted = retag_group(candidate, "voucher_matched", reason)
|
|
if draft_no:
|
|
adjusted["summary"]["draft_no"] = draft_no
|
|
adjusted["summary"]["search_text"] = f"{clean(adjusted['summary'].get('search_text'))} {draft_no} {reason}".strip()
|
|
moves[id(candidate)] = (adjusted, candidate_status)
|
|
|
|
if not moves:
|
|
return
|
|
for status_key in candidate_statuses:
|
|
status_groups[status_key] = [
|
|
group for group in list(status_groups.get(status_key) or []) if id(group) not in moves
|
|
]
|
|
status_groups["voucher_matched"] = list(status_groups.get("voucher_matched") or []) + [
|
|
adjusted for adjusted, _source_status in moves.values()
|
|
]
|
|
|
|
|
|
def move_excepted_groups(status_groups: dict[str, list[dict[str, Any]]]) -> None:
|
|
excepted: list[dict[str, Any]] = []
|
|
recheck_from_matched: list[dict[str, Any]] = []
|
|
unmatched_from_excepted: list[dict[str, Any]] = []
|
|
for group in list(status_groups.get("voucher_excepted") or []):
|
|
existing_reason = clean((group.get("summary") or {}).get("review_reason"))
|
|
if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES:
|
|
excepted.append(mark_excepted(group, "MANUAL_OFFSET_PAIR_EXCEPTED"))
|
|
continue
|
|
if "WEHAGO_EXCEPTED_OFFSET" in existing_reason or "WEHAGO_EXCEPTED_CONFIRMED_REVERSAL_PAIR" in existing_reason:
|
|
excepted.append(mark_excepted(group, ""))
|
|
continue
|
|
is_excepted, reason = _is_wehago_excepted_voucher_group(group)
|
|
if is_excepted:
|
|
excepted.append(mark_excepted(group, reason))
|
|
else:
|
|
recheck_from_matched.append(
|
|
retag_group(group, "voucher_recheck", "PROJECTION_RECONCILE_EXCEPTED_RULE_RELAXED_RECHECK")
|
|
)
|
|
retained_by_status: dict[str, list[dict[str, Any]]] = {}
|
|
for status_key in ("voucher_matched", "voucher_unmatched", "voucher_recheck"):
|
|
retained: list[dict[str, Any]] = []
|
|
for group in list(status_groups.get(status_key) or []):
|
|
if group_identity(group) in MANUAL_OFFSET_EXCEPTED_IDENTITIES:
|
|
excepted.append(mark_excepted(group, "MANUAL_OFFSET_PAIR_EXCEPTED"))
|
|
continue
|
|
is_excepted, reason = _is_wehago_excepted_voucher_group(group)
|
|
if is_excepted and status_key == "voucher_matched" and reason != "WEHAGO_EXCEPTED_DEPRECIATION_ACCOUNT":
|
|
recheck_from_matched.append(
|
|
retag_group(group, "voucher_recheck", f"PROJECTION_RECONCILE_EXCEPTED_CANDIDATE_RECHECK / {reason}")
|
|
)
|
|
elif is_excepted:
|
|
excepted.append(mark_excepted(group, reason))
|
|
else:
|
|
retained.append(group)
|
|
retained_by_status[status_key] = retained
|
|
retained_by_status["voucher_unmatched"].extend(unmatched_from_excepted)
|
|
retained_by_status["voucher_recheck"].extend(recheck_from_matched)
|
|
confirmed = _move_wehago_confirmed_reversal_pairs_to_excepted(
|
|
{
|
|
"voucher_matched": retained_by_status["voucher_matched"],
|
|
"voucher_unmatched": retained_by_status["voucher_unmatched"],
|
|
"voucher_recheck": retained_by_status["voucher_recheck"],
|
|
"voucher_excepted": excepted,
|
|
}
|
|
)
|
|
retained_by_status["voucher_matched"] = list(confirmed.get("voucher_matched") or [])
|
|
retained_by_status["voucher_unmatched"] = list(confirmed.get("voucher_unmatched") or [])
|
|
retained_by_status["voucher_recheck"] = list(confirmed.get("voucher_recheck") or [])
|
|
excepted = list(confirmed.get("voucher_excepted") or [])
|
|
apply_net_adjustment_component_matches(retained_by_status)
|
|
moved = _move_wehago_offset_tax_invoice_groups_to_excepted(
|
|
{
|
|
"voucher_unmatched": retained_by_status["voucher_unmatched"],
|
|
"voucher_recheck": retained_by_status["voucher_recheck"],
|
|
"voucher_excepted": excepted,
|
|
}
|
|
)
|
|
status_groups["voucher_matched"] = retained_by_status["voucher_matched"]
|
|
status_groups["voucher_unmatched"] = list(moved.get("voucher_unmatched") or [])
|
|
status_groups["voucher_recheck"] = list(moved.get("voucher_recheck") or [])
|
|
status_groups["voucher_excepted"] = [
|
|
mark_excepted(group, "") for group in list(moved.get("voucher_excepted") or [])
|
|
]
|
|
|
|
|
|
def append_unique(values: list[str], value: Any) -> None:
|
|
text = clean(value)
|
|
if text and text not in values:
|
|
values.append(text)
|
|
|
|
|
|
def rebuild_summary(group: dict[str, Any], status_key: str) -> dict[str, Any]:
|
|
old = dict(group.get("summary") or {})
|
|
rows = list(group.get("rows") or [])
|
|
ledger_accounts: list[str] = []
|
|
voucher_accounts: list[str] = []
|
|
ledger_vendors: list[str] = []
|
|
voucher_vendors: list[str] = []
|
|
draft_nos: list[str] = []
|
|
reasons: list[str] = []
|
|
summary = {
|
|
**old,
|
|
"ledger_row_count": 0,
|
|
"voucher_row_count": 0,
|
|
"ledger_debit": 0.0,
|
|
"ledger_credit": 0.0,
|
|
"voucher_debit": 0.0,
|
|
"voucher_credit": 0.0,
|
|
}
|
|
for row in rows:
|
|
if clean(row.get("ledger_account_name")):
|
|
summary["ledger_row_count"] += 1
|
|
summary["ledger_debit"] += parse_amount(row.get("ledger_debit"))
|
|
summary["ledger_credit"] += parse_amount(row.get("ledger_credit"))
|
|
append_unique(ledger_accounts, row.get("ledger_account_name"))
|
|
append_unique(ledger_vendors, row.get("ledger_vendor"))
|
|
if clean(row.get("voucher_account_name")):
|
|
summary["voucher_row_count"] += 1
|
|
summary["voucher_debit"] += parse_amount(row.get("voucher_debit"))
|
|
summary["voucher_credit"] += parse_amount(row.get("voucher_credit"))
|
|
append_unique(voucher_accounts, row.get("voucher_account_name"))
|
|
append_unique(voucher_vendors, row.get("voucher_vendor"))
|
|
append_unique(draft_nos, row.get("draft_no"))
|
|
append_unique(reasons, row.get("review_reason"))
|
|
if not clean(summary.get("ledger_date")) and clean(row.get("ledger_date")):
|
|
summary["ledger_date"] = clean(row.get("ledger_date"))
|
|
if not clean(summary.get("proof_date")) and clean(row.get("proof_date")):
|
|
summary["proof_date"] = clean(row.get("proof_date"))
|
|
summary["status_label"] = "Matched" if status_key in {"voucher_matched", "erp_voucher_matched"} else "Recheck" if status_key == "voucher_recheck" else "Unmatched"
|
|
summary["ledger_accounts"] = ", ".join(ledger_accounts)
|
|
summary["voucher_accounts"] = ", ".join(voucher_accounts)
|
|
summary["ledger_vendors"] = ", ".join(ledger_vendors)
|
|
summary["voucher_vendors"] = ", ".join(voucher_vendors)
|
|
summary["draft_no"] = ", ".join(draft_nos) or clean(old.get("draft_no"))
|
|
summary["review_reason"] = " / ".join(reasons) or clean(old.get("review_reason"))
|
|
summary["search_text"] = " ".join(
|
|
clean(part)
|
|
for part in [
|
|
summary.get("voucher_no"),
|
|
summary.get("draft_no"),
|
|
summary.get("ledger_accounts"),
|
|
summary.get("voucher_accounts"),
|
|
summary.get("ledger_vendors"),
|
|
summary.get("voucher_vendors"),
|
|
summary.get("review_reason"),
|
|
]
|
|
if clean(part)
|
|
)
|
|
return summary
|
|
|
|
|
|
def group_quality(group: dict[str, Any], status_key: str) -> tuple[int, int, float, int]:
|
|
priority = {"voucher_excepted": 4, "voucher_matched": 3, "voucher_recheck": 2, "voucher_unmatched": 1}.get(status_key, 0)
|
|
rows = list(group.get("rows") or [])
|
|
direct_rows = sum(1 for row in rows if has_wehago_value(row) and has_erp_value(row))
|
|
amount = max(
|
|
parse_amount((group.get("summary") or {}).get("ledger_debit")),
|
|
parse_amount((group.get("summary") or {}).get("ledger_credit")),
|
|
parse_amount((group.get("summary") or {}).get("voucher_debit")),
|
|
parse_amount((group.get("summary") or {}).get("voucher_credit")),
|
|
)
|
|
return priority, direct_rows, amount, -len(rows)
|
|
|
|
|
|
def build_missing_db_group(db_row: dict[str, Any], ledger_rows: list[sqlite3.Row]) -> dict[str, Any]:
|
|
key = clean(db_row.get("voucher_no"))
|
|
rows: list[dict[str, Any]] = []
|
|
for index, row in enumerate(ledger_rows):
|
|
rows.append(
|
|
{
|
|
"fiscal_year": YEAR,
|
|
"status_label": "Unmatched",
|
|
"ledger_date": clean(row["ledger_date"]) or display_date_from_db_key(key),
|
|
"proof_date": "",
|
|
"voucher_no": display_voucher_from_db_key(key),
|
|
"draft_no": "",
|
|
"ledger_account_name": clean(row["account_name"]),
|
|
"voucher_account_name": "",
|
|
"ledger_vendor": clean(row["vendor_name"]),
|
|
"voucher_vendor": "",
|
|
"ledger_debit": parse_amount(row["debit"]),
|
|
"ledger_credit": parse_amount(row["credit"]),
|
|
"voucher_debit": 0,
|
|
"voucher_credit": 0,
|
|
"ledger_desc": clean(row["description"]),
|
|
"voucher_desc": "",
|
|
"review_reason": "PROJECTION_RECONCILE_DB_ONLY_WEHAGO",
|
|
"matched_case": "",
|
|
"ledger_row_key": f"ledger:{row['id']}",
|
|
"voucher_row_key": "",
|
|
"match_identity_key": "",
|
|
}
|
|
)
|
|
if not rows:
|
|
rows.append(
|
|
{
|
|
"fiscal_year": YEAR,
|
|
"status_label": "Unmatched",
|
|
"ledger_date": display_date_from_db_key(key),
|
|
"proof_date": "",
|
|
"voucher_no": display_voucher_from_db_key(key),
|
|
"draft_no": "",
|
|
"ledger_account_name": clean(db_row.get("ledger_accounts")),
|
|
"voucher_account_name": "",
|
|
"ledger_vendor": clean(db_row.get("ledger_vendors")),
|
|
"voucher_vendor": "",
|
|
"ledger_debit": parse_amount(db_row.get("ledger_debit")),
|
|
"ledger_credit": parse_amount(db_row.get("ledger_credit")),
|
|
"voucher_debit": 0,
|
|
"voucher_credit": 0,
|
|
"ledger_desc": clean(db_row.get("notes")),
|
|
"voucher_desc": "",
|
|
"review_reason": "PROJECTION_RECONCILE_DB_ONLY_WEHAGO",
|
|
"matched_case": "",
|
|
"ledger_row_key": "",
|
|
"voucher_row_key": "",
|
|
"match_identity_key": "",
|
|
}
|
|
)
|
|
summary = {
|
|
"fiscal_year": YEAR,
|
|
"status_label": "Unmatched",
|
|
"ledger_date": display_date_from_db_key(key),
|
|
"proof_date": "",
|
|
"voucher_no": display_voucher_from_db_key(key),
|
|
"draft_no": "",
|
|
"ledger_row_count": 0,
|
|
"voucher_row_count": 0,
|
|
"ledger_debit": 0.0,
|
|
"ledger_credit": 0.0,
|
|
"voucher_debit": 0.0,
|
|
"voucher_credit": 0.0,
|
|
"ledger_accounts": "",
|
|
"voucher_accounts": "",
|
|
"ledger_vendors": "",
|
|
"voucher_vendors": "",
|
|
"review_reason": "PROJECTION_RECONCILE_DB_ONLY_WEHAGO",
|
|
"search_text": "",
|
|
}
|
|
group = {"summary": summary, "rows": rows, "source_group_index": 0}
|
|
group["summary"] = rebuild_summary(group, "voucher_unmatched")
|
|
return group
|
|
|
|
|
|
def ledger_db_row_identity(row: sqlite3.Row) -> tuple[Any, ...]:
|
|
return (
|
|
"side",
|
|
clean(row["account_name"]),
|
|
clean(row["vendor_name"]),
|
|
amount_key(row["debit"]),
|
|
amount_key(row["credit"]),
|
|
clean(row["description"]),
|
|
)
|
|
|
|
|
|
def build_unmatched_ledger_row(row: sqlite3.Row, db_key: str, reason: str) -> dict[str, Any]:
|
|
return {
|
|
"fiscal_year": YEAR,
|
|
"status_label": "Unmatched",
|
|
"ledger_date": clean(row["ledger_date"]) or display_date_from_db_key(db_key),
|
|
"proof_date": "",
|
|
"voucher_no": display_voucher_from_db_key(db_key),
|
|
"draft_no": "",
|
|
"ledger_account_name": clean(row["account_name"]),
|
|
"voucher_account_name": "",
|
|
"ledger_vendor": clean(row["vendor_name"]),
|
|
"voucher_vendor": "",
|
|
"ledger_debit": parse_amount(row["debit"]),
|
|
"ledger_credit": parse_amount(row["credit"]),
|
|
"voucher_debit": 0,
|
|
"voucher_credit": 0,
|
|
"ledger_desc": clean(row["description"]),
|
|
"voucher_desc": "",
|
|
"review_reason": reason,
|
|
"matched_case": "",
|
|
"ledger_row_key": f"ledger:{row['id']}",
|
|
"voucher_row_key": "",
|
|
"match_identity_key": "",
|
|
}
|
|
|
|
|
|
def supplement_group_with_missing_wehago_rows(
|
|
group: dict[str, Any],
|
|
status_key: str,
|
|
db_key: str,
|
|
ledger_rows: list[sqlite3.Row],
|
|
) -> dict[str, Any]:
|
|
if status_key not in {"voucher_matched", "voucher_recheck"} or not ledger_rows:
|
|
return group
|
|
rows = [dict(row) for row in group.get("rows") or []]
|
|
seen = {
|
|
ledger_row_identity(row)
|
|
for row in rows
|
|
if clean(row.get("ledger_account_name"))
|
|
}
|
|
added = False
|
|
for ledger_row in ledger_rows:
|
|
identity = ledger_db_row_identity(ledger_row)
|
|
if identity in seen:
|
|
continue
|
|
rows.append(build_unmatched_ledger_row(ledger_row, db_key, "PROJECTION_RECONCILE_PARTIAL_MATCH_WEHAGO_ROW_UNMATCHED"))
|
|
seen.add(identity)
|
|
added = True
|
|
if not added:
|
|
return group
|
|
supplemented = {
|
|
"summary": dict(group.get("summary") or {}),
|
|
"rows": rows,
|
|
"source_group_index": group.get("source_group_index"),
|
|
}
|
|
supplemented["summary"] = rebuild_summary(supplemented, status_key)
|
|
return supplemented
|
|
|
|
|
|
def insert_group(conn: sqlite3.Connection, signature: str, status_key: str, group_index: int, group: dict[str, Any]) -> None:
|
|
summary = dict(group["summary"])
|
|
summary.update(
|
|
{
|
|
"start_year": YEAR,
|
|
"end_year": YEAR,
|
|
"status_key": status_key,
|
|
"signature": signature,
|
|
"group_index": group_index,
|
|
}
|
|
)
|
|
conn.execute(
|
|
f"""
|
|
INSERT INTO wehago_compare_query_groups ({', '.join(GROUP_COLUMNS)}, created_at, updated_at)
|
|
VALUES ({', '.join('?' for _ in GROUP_COLUMNS)}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
""",
|
|
[summary.get(column, "") for column in GROUP_COLUMNS],
|
|
)
|
|
for row_index, row in enumerate(group.get("rows") or []):
|
|
row = dict(row)
|
|
row.update(
|
|
{
|
|
"start_year": YEAR,
|
|
"end_year": YEAR,
|
|
"status_key": status_key,
|
|
"signature": signature,
|
|
"group_index": group_index,
|
|
"row_index": row_index,
|
|
}
|
|
)
|
|
conn.execute(
|
|
f"""
|
|
INSERT INTO wehago_compare_query_rows ({', '.join(ROW_COLUMNS)}, created_at, updated_at)
|
|
VALUES ({', '.join('?' for _ in ROW_COLUMNS)}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
""",
|
|
[row.get(column, "") for column in ROW_COLUMNS],
|
|
)
|
|
|
|
|
|
def update_snapshot_row_counts(conn: sqlite3.Connection, counts: dict[str, int]) -> None:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT row_counts_json
|
|
FROM wehago_snapshot_status
|
|
WHERE fiscal_year = ?
|
|
LIMIT 1
|
|
""",
|
|
(YEAR,),
|
|
).fetchone()
|
|
merged: dict[str, int] = {}
|
|
if row:
|
|
try:
|
|
parsed = json.loads(row["row_counts_json"] or "{}")
|
|
if isinstance(parsed, dict):
|
|
merged = {str(key): int(value or 0) for key, value in parsed.items()}
|
|
except Exception:
|
|
merged = {}
|
|
for status_key, count in counts.items():
|
|
merged[status_key] = int(count or 0)
|
|
payload = json.dumps(merged, ensure_ascii=False)
|
|
if row:
|
|
conn.execute(
|
|
"""
|
|
UPDATE wehago_snapshot_status
|
|
SET state = 'ready',
|
|
row_counts_json = ?,
|
|
updated_at = CURRENT_TIMESTAMP,
|
|
last_built_at = CURRENT_TIMESTAMP,
|
|
error_message = ''
|
|
WHERE fiscal_year = ?
|
|
""",
|
|
(payload, YEAR),
|
|
)
|
|
else:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO wehago_snapshot_status (
|
|
fiscal_year, snapshot_signature, state, row_counts_json,
|
|
error_message, created_at, updated_at, last_requested_at, last_built_at
|
|
)
|
|
VALUES (?, ?, 'ready', ?, '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
""",
|
|
(YEAR, "", payload),
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
global MANUAL_OFFSET_EXCEPTED_IDENTITIES, RAW_ERP_ROWS_BY_DRAFT_BASE, YEAR
|
|
parser = argparse.ArgumentParser(description="Reconcile a WEHAGO comparison query projection to DB rows.")
|
|
parser.add_argument("--year", type=int, default=YEAR)
|
|
args = parser.parse_args()
|
|
YEAR = int(args.year)
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
conn.row_factory = sqlite3.Row
|
|
cur = conn.cursor()
|
|
source_signature = latest_projection_signature(cur)
|
|
reconciled_prefix = f"{QUERY_PROJECTION_VERSION}|db-reconciled-v1|"
|
|
target_signature = source_signature if source_signature.startswith(reconciled_prefix) else f"{reconciled_prefix}{source_signature}"
|
|
RAW_ERP_ROWS_BY_DRAFT_BASE = load_raw_erp_rows_by_draft_base(conn)
|
|
MANUAL_OFFSET_EXCEPTED_IDENTITIES = load_manual_offset_excepted_identities(conn)
|
|
db_wehago = load_db_wehago(conn)
|
|
groups = load_groups(conn, source_signature)
|
|
|
|
ledger_rows_by_key: dict[str, list[sqlite3.Row]] = defaultdict(list)
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT *
|
|
FROM wehago_ledger_rows
|
|
WHERE fiscal_year = ? AND COALESCE(compare_voucher_no, '') <> ''
|
|
ORDER BY compare_voucher_no, row_number
|
|
""",
|
|
(YEAR,),
|
|
).fetchall():
|
|
ledger_rows_by_key[clean(row["compare_voucher_no"])].append(row)
|
|
|
|
selected_by_key: dict[str, tuple[str, dict[str, Any]]] = {}
|
|
duplicate_counter = 0
|
|
removed_non_db = 0
|
|
for status_key in WEHAGO_STATUSES:
|
|
for group in groups.get(status_key, []):
|
|
for identity, split_group in split_group_by_wehago_voucher(group, status_key).items():
|
|
split_group = clean_group_rows(split_group, status_key)
|
|
if not identity or identity not in db_wehago:
|
|
removed_non_db += 1
|
|
continue
|
|
final_status_key = normalized_wehago_status(status_key, split_group)
|
|
if final_status_key != status_key:
|
|
split_group["summary"] = rebuild_summary(split_group, final_status_key)
|
|
current = selected_by_key.get(identity)
|
|
if current is None or group_quality(split_group, final_status_key) > group_quality(current[1], current[0]):
|
|
if current is not None:
|
|
duplicate_counter += 1
|
|
selected_by_key[identity] = (final_status_key, split_group)
|
|
else:
|
|
duplicate_counter += 1
|
|
|
|
missing_db = sorted(set(db_wehago) - set(selected_by_key))
|
|
for key in missing_db:
|
|
selected_by_key[key] = (
|
|
"voucher_unmatched",
|
|
build_missing_db_group(db_wehago[key], ledger_rows_by_key.get(key, [])),
|
|
)
|
|
|
|
status_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for key in sorted(selected_by_key):
|
|
status_key, group = selected_by_key[key]
|
|
group = supplement_group_with_missing_wehago_rows(
|
|
group,
|
|
status_key,
|
|
key,
|
|
ledger_rows_by_key.get(key, []),
|
|
)
|
|
status_groups[status_key].append(group)
|
|
|
|
move_excepted_groups(status_groups)
|
|
excepted_wehago_keys = {
|
|
group_identity(group)
|
|
for group in status_groups.get("voucher_excepted") or []
|
|
if group_identity(group)
|
|
}
|
|
allowed_drafts_by_wehago = {
|
|
key: allowed_erp_drafts_for_group(group.get("rows") or [])
|
|
for group in status_groups.get("voucher_matched") or []
|
|
for key in [group_identity(group)]
|
|
if key
|
|
}
|
|
for status_key in ERP_STATUSES:
|
|
seen_erp: set[str] = set()
|
|
for group in groups.get(status_key, []):
|
|
cleaned = clean_group_rows(group, status_key)
|
|
if status_key == "erp_voucher_matched":
|
|
normalized_rows: list[dict[str, Any]] = []
|
|
for row in cleaned.get("rows") or []:
|
|
identity = row_wehago_identity(row, cleaned)
|
|
draft = erp_voucher_base(row.get("draft_no"))
|
|
allowed_drafts = allowed_drafts_by_wehago.get(identity)
|
|
if has_wehago_value(row) and has_erp_value(row) and not direct_row_matches_account_nature(row):
|
|
normalized_rows.append(blank_wehago_side(row, "PROJECTION_RECONCILE_INVALID_ACCOUNT_NATURE"))
|
|
continue
|
|
if has_wehago_value(row) and identity in excepted_wehago_keys:
|
|
if has_erp_value(row):
|
|
normalized_rows.append(blank_wehago_side(row, "PROJECTION_RECONCILE_WEHAGO_EXCEPTED"))
|
|
continue
|
|
if has_wehago_value(row) and allowed_drafts is not None and draft not in allowed_drafts:
|
|
if has_erp_value(row):
|
|
normalized_rows.append(blank_wehago_side(row, "PROJECTION_RECONCILE_UNMATCHED_WEHAGO_OVER_CAP"))
|
|
continue
|
|
normalized_rows.append(row)
|
|
cleaned["rows"] = normalized_rows
|
|
final_status_key = (
|
|
"erp_voucher_matched"
|
|
if any(has_wehago_value(row) and has_erp_value(row) for row in normalized_rows)
|
|
else "erp_voucher_unmatched"
|
|
)
|
|
cleaned["summary"] = rebuild_summary(cleaned, final_status_key)
|
|
else:
|
|
final_status_key = status_key
|
|
summary = cleaned.get("summary") or {}
|
|
identity = "|".join(
|
|
clean(part)
|
|
for part in (summary.get("fiscal_year"), summary.get("proof_date"), erp_voucher_base(summary.get("draft_no")) or summary.get("voucher_no"))
|
|
if clean(part)
|
|
)
|
|
if not identity or identity in seen_erp:
|
|
continue
|
|
seen_erp.add(identity)
|
|
status_groups[final_status_key].append(cleaned)
|
|
|
|
conn.execute("BEGIN")
|
|
try:
|
|
conn.execute(
|
|
"DELETE FROM wehago_compare_query_rows WHERE start_year = ? AND end_year = ? AND signature = ?",
|
|
(YEAR, YEAR, target_signature),
|
|
)
|
|
conn.execute(
|
|
"DELETE FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ?",
|
|
(YEAR, YEAR, target_signature),
|
|
)
|
|
counts: dict[str, int] = {}
|
|
for status_key in ALL_VOUCHER_STATUSES:
|
|
for group_index, group in enumerate(status_groups.get(status_key, []), start=1):
|
|
insert_group(conn, target_signature, status_key, group_index, group)
|
|
counts[status_key] = len(status_groups.get(status_key, []))
|
|
update_snapshot_row_counts(conn, counts)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO wehago_action_history (action_type, payload_json, created_at)
|
|
VALUES ('reconcile_wehago_projection', ?, CURRENT_TIMESTAMP)
|
|
""",
|
|
(
|
|
json.dumps(
|
|
{
|
|
"start_year": YEAR,
|
|
"end_year": YEAR,
|
|
"signature": target_signature,
|
|
"source_signature": source_signature,
|
|
"db_wehago_count": len(db_wehago),
|
|
"counts": counts,
|
|
"removed_duplicate_groups": duplicate_counter,
|
|
"removed_non_db_groups": removed_non_db,
|
|
"added_db_only_groups": len(missing_db),
|
|
"created_at": datetime.now().isoformat(timespec="seconds"),
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
),
|
|
)
|
|
conn.commit()
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"source_signature": source_signature,
|
|
"target_signature": target_signature,
|
|
"db_wehago_count": len(db_wehago),
|
|
"counts": counts,
|
|
"wehago_sum": sum(counts.get(status, 0) for status in WEHAGO_STATUSES),
|
|
"removed_duplicate_groups": duplicate_counter,
|
|
"removed_non_db_groups": removed_non_db,
|
|
"added_db_only_groups": len(missing_db),
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|