Update WEHAGO comparison data and tools

2026-05-09 02:49:34 +09:00
parent b0b4647bb4
commit 7ceee2f897
44 changed files with 6296 additions and 135 deletions
@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from openpyxl import load_workbook
+from sqlalchemy import bindparam, create_engine, text
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+if str(BASE_DIR) not in sys.path:
+    sys.path.insert(0, str(BASE_DIR))
+
+from wehago_compare import (
+    WEHAGO_SOURCE_ROOT,
+    _fallback_metric_counts_from_db,
+    compute_file_hash,
+    detect_file_kind,
+    import_ledger_rows,
+    import_voucher_rows,
+    infer_year_hint,
+    init_wehago_compare_db,
+    rebuild_comparison_results,
+    upsert_source_file,
+)
+
+
+DB_PATH = BASE_DIR / "data.db"
+ENGINE = create_engine(f"sqlite:///{DB_PATH}", connect_args={"check_same_thread": False})
+
+LEDGER_FILES = {
+    2022: WEHAGO_SOURCE_ROOT / "data_download/2022/ledger_2022_20220101_20221231_260507.xlsx",
+    2023: WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260506_2023_compare/ledger_2023_20230101_20231231_260506.xlsx",
+    2024: WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260506_2024_compare/ledger_2024_20240101_20241231_260504.xlsx",
+    2025: WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260423/ledger_2025_20250101_20251231_260423.xlsx",
+}
+VOUCHER_FILES = [
+    {
+        "path": WEHAGO_SOURCE_ROOT / "voucher_sort_20220101_20221231_260507.xlsx",
+        "expected_year": 2022,
+        "year_hint": 2022,
+        "allowed_years": {2022},
+    },
+    {
+        "path": WEHAGO_SOURCE_ROOT / "voucher_sort_20230101_20251231_260424.xlsx",
+        "expected_year": 2023,
+        "year_hint": None,
+        "allowed_years": {2023, 2024},
+    },
+    {
+        "path": WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260423/voucher_2025_20250101_20251231_260422.xlsx",
+        "expected_year": 2025,
+        "year_hint": 2025,
+        "allowed_years": {2025},
+    },
+]
+_DEFAULT_YEAR_HINT = object()
+
+
+def _load_sheet(path: Path):
+    workbook = load_workbook(path, read_only=True, data_only=True)
+    return workbook, workbook.worksheets[0]
+
+
+def _delete_year_data(conn, *, ledger_years: set[int], voucher_years: set[int]) -> dict[str, int]:
+    ledger_deleted = conn.execute(
+        text("DELETE FROM wehago_ledger_rows WHERE fiscal_year IN :years").bindparams(bindparam("years", expanding=True)),
+        {"years": sorted(ledger_years)},
+    ).rowcount
+    voucher_deleted = conn.execute(
+        text("DELETE FROM wehago_voucher_rows WHERE fiscal_year IN :years").bindparams(bindparam("years", expanding=True)),
+        {"years": sorted(voucher_years)},
+    ).rowcount
+
+    unused_source_ids = {
+        int(row[0])
+        for row in conn.execute(
+            text(
+                """
+                SELECT sf.id
+                FROM wehago_source_files sf
+                LEFT JOIN wehago_ledger_rows l ON l.source_file_id = sf.id
+                LEFT JOIN wehago_voucher_rows v ON v.source_file_id = sf.id
+                GROUP BY sf.id
+                HAVING COUNT(l.id) = 0 AND COUNT(v.id) = 0
+                """
+            )
+        )
+    }
+    if unused_source_ids:
+        conn.execute(
+            text("DELETE FROM wehago_source_files WHERE id IN :ids").bindparams(bindparam("ids", expanding=True)),
+            {"ids": sorted(unused_source_ids)},
+        )
+    conn.execute(
+        text("DELETE FROM wehago_comparison_results WHERE fiscal_year IN :years").bindparams(bindparam("years", expanding=True)),
+        {"years": sorted(ledger_years | voucher_years)},
+    )
+    conn.execute(text("DELETE FROM wehago_metric_count_cache"))
+    conn.execute(text("DELETE FROM wehago_result_row_cache"))
+    conn.execute(text("DELETE FROM wehago_pair_recommend_cache"))
+    conn.execute(text("DELETE FROM wehago_background_jobs WHERE job_type = 'pair_recommend_precompute'"))
+    return {"ledger_deleted": int(ledger_deleted or 0), "voucher_deleted": int(voucher_deleted or 0), "source_deleted": len(unused_source_ids)}
+
+
+def _import_one(
+    conn,
+    path: Path,
+    expected_kind: str,
+    expected_year: int,
+    *,
+    allowed_years: set[int] | None = None,
+    year_hint_override: int | None | object = _DEFAULT_YEAR_HINT,
+) -> dict[str, object]:
+    file_kind, header, sheet_name = detect_file_kind(path)
+    if file_kind != expected_kind:
+        raise ValueError(f"{path} 형식이 {expected_kind}가 아닙니다: {file_kind}")
+
+    workbook, sheet = _load_sheet(path)
+    try:
+        sample_rows = list(sheet.iter_rows(min_row=2, max_row=51, values_only=True))
+        if year_hint_override is _DEFAULT_YEAR_HINT:
+            year_hint = infer_year_hint(path, file_kind, sample_rows) or expected_year
+        else:
+            year_hint = year_hint_override
+        source_id, _changed = upsert_source_file(conn, path, file_kind, year_hint, sheet_name, header)
+        if file_kind == "ledger":
+            conn.execute(text("DELETE FROM wehago_ledger_rows WHERE source_file_id = :source_id"), {"source_id": source_id})
+            inserted = import_ledger_rows(conn, source_id, sheet_name, sheet.iter_rows(min_row=2, values_only=True), year_hint)
+        else:
+            conn.execute(text("DELETE FROM wehago_voucher_rows WHERE source_file_id = :source_id"), {"source_id": source_id})
+            inserted = import_voucher_rows(conn, source_id, sheet_name, sheet.iter_rows(min_row=2, values_only=True), year_hint)
+        allowed = sorted(allowed_years or {expected_year})
+        if allowed:
+            conn.execute(
+                text(
+                    """
+                    DELETE FROM wehago_ledger_rows
+                    WHERE source_file_id = :source_id
+                      AND fiscal_year NOT IN :allowed_years
+                    """
+                ).bindparams(bindparam("allowed_years", expanding=True)),
+                {"source_id": source_id, "allowed_years": allowed},
+            )
+            conn.execute(
+                text(
+                    """
+                    DELETE FROM wehago_voucher_rows
+                    WHERE source_file_id = :source_id
+                      AND fiscal_year NOT IN :allowed_years
+                    """
+                ).bindparams(bindparam("allowed_years", expanding=True)),
+                {"source_id": source_id, "allowed_years": allowed},
+            )
+            inserted = int(
+                conn.execute(
+                    text(f"SELECT COUNT(*) FROM wehago_{file_kind}_rows WHERE source_file_id = :source_id"),
+                    {"source_id": source_id},
+                ).scalar_one()
+                or 0
+            )
+        conn.execute(
+            text("UPDATE wehago_source_files SET row_count = :row_count, imported_at = CURRENT_TIMESTAMP WHERE id = :source_id"),
+            {"row_count": inserted, "source_id": source_id},
+        )
+        return {
+            "path": str(path),
+            "kind": file_kind,
+            "year_hint": year_hint,
+            "allowed_years": allowed,
+            "source_id": source_id,
+            "inserted": inserted,
+            "file_hash": compute_file_hash(path),
+        }
+    finally:
+        workbook.close()
+
+
+def _fetch_counts(conn) -> dict[str, object]:
+    counts: dict[str, object] = {}
+    for table in ("wehago_ledger_rows", "wehago_voucher_rows", "wehago_comparison_results"):
+        rows = conn.execute(
+            text(f"SELECT fiscal_year, COUNT(*) AS row_count FROM {table} GROUP BY fiscal_year ORDER BY fiscal_year")
+        ).mappings()
+        counts[table] = {str(row["fiscal_year"]): int(row["row_count"]) for row in rows}
+    return counts
+
+
+def _fetch_quality(conn) -> dict[str, object]:
+    total_rows = conn.execute(
+        text(
+            """
+            SELECT COUNT(*)
+            FROM wehago_ledger_rows
+            WHERE REPLACE(COALESCE(description, ''), ' ', '') IN ('[월계]', '[누계]', '월계', '누계')
+               OR REPLACE(COALESCE(ledger_date, ''), ' ', '') IN ('[월계]', '[누계]', '월계', '누계')
+            """
+        )
+    ).scalar_one()
+    blank_account_rows = conn.execute(
+        text(
+            """
+            SELECT COUNT(*)
+            FROM wehago_ledger_rows
+            WHERE fiscal_year IN (2022, 2023, 2024)
+              AND (COALESCE(account_code, '') = '' OR COALESCE(account_name, '') = '')
+            """
+        )
+    ).scalar_one()
+    account_names = conn.execute(
+        text(
+            """
+            SELECT fiscal_year, account_code, COUNT(DISTINCT account_name) AS name_count,
+                   GROUP_CONCAT(DISTINCT account_name) AS names
+            FROM wehago_ledger_rows
+            WHERE fiscal_year IN (2022, 2023, 2024)
+              AND COALESCE(account_code, '') <> ''
+            GROUP BY fiscal_year, account_code
+            HAVING COUNT(DISTINCT account_name) > 1
+            ORDER BY fiscal_year, account_code
+            LIMIT 20
+            """
+        )
+    ).mappings()
+    voucher_counts_by_source = conn.execute(
+        text(
+            """
+            SELECT sf.file_name, v.fiscal_year, COUNT(*) AS row_count
+            FROM wehago_voucher_rows v
+            JOIN wehago_source_files sf ON sf.id = v.source_file_id
+            WHERE v.fiscal_year IN (2022, 2023, 2024, 2025)
+            GROUP BY sf.file_name, v.fiscal_year
+            ORDER BY v.fiscal_year, sf.file_name
+            """
+        )
+    ).mappings()
+    return {
+        "monthly_cumulative_rows": int(total_rows or 0),
+        "blank_account_rows_2022_2024": int(blank_account_rows or 0),
+        "multi_name_accounts": [dict(row) for row in account_names],
+        "voucher_counts_by_source": [dict(row) for row in voucher_counts_by_source],
+    }
+
+
+def main() -> None:
+    voucher_paths = [item["path"] for item in VOUCHER_FILES]
+    missing = [path for path in [*LEDGER_FILES.values(), *voucher_paths] if not path.exists()]
+    if missing:
+        raise FileNotFoundError("\\n".join(str(path) for path in missing))
+
+    init_wehago_compare_db(ENGINE)
+    summary: dict[str, object] = {"deleted": {}, "imported": []}
+    with ENGINE.begin() as conn:
+        summary["deleted"] = _delete_year_data(conn, ledger_years={2022, 2023, 2024, 2025}, voucher_years={2022, 2023, 2024, 2025})
+        for year, path in LEDGER_FILES.items():
+            summary["imported"].append(_import_one(conn, path, "ledger", year, allowed_years={year}))
+        for item in VOUCHER_FILES:
+            summary["imported"].append(
+                _import_one(
+                    conn,
+                    item["path"],
+                    "voucher",
+                    int(item["expected_year"]),
+                    allowed_years=set(item["allowed_years"]),
+                    year_hint_override=item["year_hint"],
+                )
+            )
+        rebuild_comparison_results(conn)
+        summary["dashboard_counts"] = {
+            str(year): _fallback_metric_counts_from_db(conn, year, year)
+            for year in range(2022, 2026)
+        }
+        summary["counts"] = _fetch_counts(conn)
+        summary["quality"] = _fetch_quality(conn)
+
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()