282 lines
11 KiB
Python
282 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from openpyxl import load_workbook
|
|
from sqlalchemy import bindparam, create_engine, text
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
|
if str(BASE_DIR) not in sys.path:
|
|
sys.path.insert(0, str(BASE_DIR))
|
|
|
|
from wehago_compare import (
|
|
WEHAGO_SOURCE_ROOT,
|
|
_fallback_metric_counts_from_db,
|
|
compute_file_hash,
|
|
detect_file_kind,
|
|
import_ledger_rows,
|
|
import_voucher_rows,
|
|
infer_year_hint,
|
|
init_wehago_compare_db,
|
|
rebuild_comparison_results,
|
|
upsert_source_file,
|
|
)
|
|
from runtime_config import DB_PATH
|
|
|
|
|
|
ENGINE = create_engine(f"sqlite:///{DB_PATH}", connect_args={"check_same_thread": False})
|
|
|
|
LEDGER_FILES = {
|
|
2022: WEHAGO_SOURCE_ROOT / "data_download/2022/ledger_2022_20220101_20221231_260507.xlsx",
|
|
2023: WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260506_2023_compare/ledger_2023_20230101_20231231_260506.xlsx",
|
|
2024: WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260506_2024_compare/ledger_2024_20240101_20241231_260504.xlsx",
|
|
2025: WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260423/ledger_2025_20250101_20251231_260423.xlsx",
|
|
}
|
|
VOUCHER_FILES = [
|
|
{
|
|
"path": WEHAGO_SOURCE_ROOT / "voucher_sort_20220101_20221231_260507.xlsx",
|
|
"expected_year": 2022,
|
|
"year_hint": 2022,
|
|
"allowed_years": {2022},
|
|
},
|
|
{
|
|
"path": WEHAGO_SOURCE_ROOT / "voucher_sort_20230101_20251231_260424.xlsx",
|
|
"expected_year": 2023,
|
|
"year_hint": None,
|
|
"allowed_years": {2023, 2024},
|
|
},
|
|
{
|
|
"path": WEHAGO_SOURCE_ROOT / "data_download/wehago_account_ledger_20260423/voucher_2025_20250101_20251231_260422.xlsx",
|
|
"expected_year": 2025,
|
|
"year_hint": 2025,
|
|
"allowed_years": {2025},
|
|
},
|
|
]
|
|
_DEFAULT_YEAR_HINT = object()
|
|
|
|
|
|
def _load_sheet(path: Path):
|
|
workbook = load_workbook(path, read_only=True, data_only=True)
|
|
return workbook, workbook.worksheets[0]
|
|
|
|
|
|
def _delete_year_data(conn, *, ledger_years: set[int], voucher_years: set[int]) -> dict[str, int]:
|
|
ledger_deleted = conn.execute(
|
|
text("DELETE FROM wehago_ledger_rows WHERE fiscal_year IN :years").bindparams(bindparam("years", expanding=True)),
|
|
{"years": sorted(ledger_years)},
|
|
).rowcount
|
|
voucher_deleted = conn.execute(
|
|
text("DELETE FROM wehago_voucher_rows WHERE fiscal_year IN :years").bindparams(bindparam("years", expanding=True)),
|
|
{"years": sorted(voucher_years)},
|
|
).rowcount
|
|
|
|
unused_source_ids = {
|
|
int(row[0])
|
|
for row in conn.execute(
|
|
text(
|
|
"""
|
|
SELECT sf.id
|
|
FROM wehago_source_files sf
|
|
LEFT JOIN wehago_ledger_rows l ON l.source_file_id = sf.id
|
|
LEFT JOIN wehago_voucher_rows v ON v.source_file_id = sf.id
|
|
GROUP BY sf.id
|
|
HAVING COUNT(l.id) = 0 AND COUNT(v.id) = 0
|
|
"""
|
|
)
|
|
)
|
|
}
|
|
if unused_source_ids:
|
|
conn.execute(
|
|
text("DELETE FROM wehago_source_files WHERE id IN :ids").bindparams(bindparam("ids", expanding=True)),
|
|
{"ids": sorted(unused_source_ids)},
|
|
)
|
|
conn.execute(
|
|
text("DELETE FROM wehago_comparison_results WHERE fiscal_year IN :years").bindparams(bindparam("years", expanding=True)),
|
|
{"years": sorted(ledger_years | voucher_years)},
|
|
)
|
|
conn.execute(text("DELETE FROM wehago_metric_count_cache"))
|
|
conn.execute(text("DELETE FROM wehago_result_row_cache"))
|
|
conn.execute(text("DELETE FROM wehago_pair_recommend_cache"))
|
|
conn.execute(text("DELETE FROM wehago_background_jobs WHERE job_type = 'pair_recommend_precompute'"))
|
|
return {"ledger_deleted": int(ledger_deleted or 0), "voucher_deleted": int(voucher_deleted or 0), "source_deleted": len(unused_source_ids)}
|
|
|
|
|
|
def _import_one(
|
|
conn,
|
|
path: Path,
|
|
expected_kind: str,
|
|
expected_year: int,
|
|
*,
|
|
allowed_years: set[int] | None = None,
|
|
year_hint_override: int | None | object = _DEFAULT_YEAR_HINT,
|
|
) -> dict[str, object]:
|
|
file_kind, header, sheet_name = detect_file_kind(path)
|
|
if file_kind != expected_kind:
|
|
raise ValueError(f"{path} 형식이 {expected_kind}가 아닙니다: {file_kind}")
|
|
|
|
workbook, sheet = _load_sheet(path)
|
|
try:
|
|
sample_rows = list(sheet.iter_rows(min_row=2, max_row=51, values_only=True))
|
|
if year_hint_override is _DEFAULT_YEAR_HINT:
|
|
year_hint = infer_year_hint(path, file_kind, sample_rows) or expected_year
|
|
else:
|
|
year_hint = year_hint_override
|
|
source_id, _changed = upsert_source_file(conn, path, file_kind, year_hint, sheet_name, header)
|
|
if file_kind == "ledger":
|
|
conn.execute(text("DELETE FROM wehago_ledger_rows WHERE source_file_id = :source_id"), {"source_id": source_id})
|
|
inserted = import_ledger_rows(conn, source_id, sheet_name, sheet.iter_rows(min_row=2, values_only=True), year_hint)
|
|
else:
|
|
conn.execute(text("DELETE FROM wehago_voucher_rows WHERE source_file_id = :source_id"), {"source_id": source_id})
|
|
inserted = import_voucher_rows(conn, source_id, sheet_name, sheet.iter_rows(min_row=2, values_only=True), year_hint)
|
|
allowed = sorted(allowed_years or {expected_year})
|
|
if allowed:
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
DELETE FROM wehago_ledger_rows
|
|
WHERE source_file_id = :source_id
|
|
AND fiscal_year NOT IN :allowed_years
|
|
"""
|
|
).bindparams(bindparam("allowed_years", expanding=True)),
|
|
{"source_id": source_id, "allowed_years": allowed},
|
|
)
|
|
conn.execute(
|
|
text(
|
|
"""
|
|
DELETE FROM wehago_voucher_rows
|
|
WHERE source_file_id = :source_id
|
|
AND fiscal_year NOT IN :allowed_years
|
|
"""
|
|
).bindparams(bindparam("allowed_years", expanding=True)),
|
|
{"source_id": source_id, "allowed_years": allowed},
|
|
)
|
|
inserted = int(
|
|
conn.execute(
|
|
text(f"SELECT COUNT(*) FROM wehago_{file_kind}_rows WHERE source_file_id = :source_id"),
|
|
{"source_id": source_id},
|
|
).scalar_one()
|
|
or 0
|
|
)
|
|
conn.execute(
|
|
text("UPDATE wehago_source_files SET row_count = :row_count, imported_at = CURRENT_TIMESTAMP WHERE id = :source_id"),
|
|
{"row_count": inserted, "source_id": source_id},
|
|
)
|
|
return {
|
|
"path": str(path),
|
|
"kind": file_kind,
|
|
"year_hint": year_hint,
|
|
"allowed_years": allowed,
|
|
"source_id": source_id,
|
|
"inserted": inserted,
|
|
"file_hash": compute_file_hash(path),
|
|
}
|
|
finally:
|
|
workbook.close()
|
|
|
|
|
|
def _fetch_counts(conn) -> dict[str, object]:
|
|
counts: dict[str, object] = {}
|
|
for table in ("wehago_ledger_rows", "wehago_voucher_rows", "wehago_comparison_results"):
|
|
rows = conn.execute(
|
|
text(f"SELECT fiscal_year, COUNT(*) AS row_count FROM {table} GROUP BY fiscal_year ORDER BY fiscal_year")
|
|
).mappings()
|
|
counts[table] = {str(row["fiscal_year"]): int(row["row_count"]) for row in rows}
|
|
return counts
|
|
|
|
|
|
def _fetch_quality(conn) -> dict[str, object]:
|
|
total_rows = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT COUNT(*)
|
|
FROM wehago_ledger_rows
|
|
WHERE REPLACE(COALESCE(description, ''), ' ', '') IN ('[월계]', '[누계]', '월계', '누계')
|
|
OR REPLACE(COALESCE(ledger_date, ''), ' ', '') IN ('[월계]', '[누계]', '월계', '누계')
|
|
"""
|
|
)
|
|
).scalar_one()
|
|
blank_account_rows = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT COUNT(*)
|
|
FROM wehago_ledger_rows
|
|
WHERE fiscal_year IN (2022, 2023, 2024)
|
|
AND (COALESCE(account_code, '') = '' OR COALESCE(account_name, '') = '')
|
|
"""
|
|
)
|
|
).scalar_one()
|
|
account_names = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT fiscal_year, account_code, COUNT(DISTINCT account_name) AS name_count,
|
|
GROUP_CONCAT(DISTINCT account_name) AS names
|
|
FROM wehago_ledger_rows
|
|
WHERE fiscal_year IN (2022, 2023, 2024)
|
|
AND COALESCE(account_code, '') <> ''
|
|
GROUP BY fiscal_year, account_code
|
|
HAVING COUNT(DISTINCT account_name) > 1
|
|
ORDER BY fiscal_year, account_code
|
|
LIMIT 20
|
|
"""
|
|
)
|
|
).mappings()
|
|
voucher_counts_by_source = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT sf.file_name, v.fiscal_year, COUNT(*) AS row_count
|
|
FROM wehago_voucher_rows v
|
|
JOIN wehago_source_files sf ON sf.id = v.source_file_id
|
|
WHERE v.fiscal_year IN (2022, 2023, 2024, 2025)
|
|
GROUP BY sf.file_name, v.fiscal_year
|
|
ORDER BY v.fiscal_year, sf.file_name
|
|
"""
|
|
)
|
|
).mappings()
|
|
return {
|
|
"monthly_cumulative_rows": int(total_rows or 0),
|
|
"blank_account_rows_2022_2024": int(blank_account_rows or 0),
|
|
"multi_name_accounts": [dict(row) for row in account_names],
|
|
"voucher_counts_by_source": [dict(row) for row in voucher_counts_by_source],
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
voucher_paths = [item["path"] for item in VOUCHER_FILES]
|
|
missing = [path for path in [*LEDGER_FILES.values(), *voucher_paths] if not path.exists()]
|
|
if missing:
|
|
raise FileNotFoundError("\\n".join(str(path) for path in missing))
|
|
|
|
init_wehago_compare_db(ENGINE)
|
|
summary: dict[str, object] = {"deleted": {}, "imported": []}
|
|
with ENGINE.begin() as conn:
|
|
summary["deleted"] = _delete_year_data(conn, ledger_years={2022, 2023, 2024, 2025}, voucher_years={2022, 2023, 2024, 2025})
|
|
for year, path in LEDGER_FILES.items():
|
|
summary["imported"].append(_import_one(conn, path, "ledger", year, allowed_years={year}))
|
|
for item in VOUCHER_FILES:
|
|
summary["imported"].append(
|
|
_import_one(
|
|
conn,
|
|
item["path"],
|
|
"voucher",
|
|
int(item["expected_year"]),
|
|
allowed_years=set(item["allowed_years"]),
|
|
year_hint_override=item["year_hint"],
|
|
)
|
|
)
|
|
rebuild_comparison_results(conn)
|
|
summary["dashboard_counts"] = {
|
|
str(year): _fallback_metric_counts_from_db(conn, year, year)
|
|
for year in range(2022, 2026)
|
|
}
|
|
summary["counts"] = _fetch_counts(conn)
|
|
summary["quality"] = _fetch_quality(conn)
|
|
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|