384 lines
16 KiB
Python
384 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from sqlalchemy import create_engine, text
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import scripts.wehago_data_download_2022_work as wehago
|
|
import scripts.wehago_ledger_api_download as wehago_api
|
|
from runtime_config import DB_PATH, WEHAGO_SOURCE_ROOT
|
|
from wehago_compare import (
|
|
detect_file_kind,
|
|
import_ledger_rows,
|
|
infer_year_hint,
|
|
rebuild_comparison_results,
|
|
upsert_source_file,
|
|
)
|
|
|
|
|
|
YEARS = (2022, 2023, 2024, 2025)
|
|
GISU_BY_YEAR = {2022: 27, 2023: 28, 2024: 29, 2025: 30}
|
|
HANMAC_LEDGER_URL = (
|
|
"https://smarta.wehago.com/#/smarta/account/SABK0107?sao"
|
|
"&cno=1173867&cd_com=biz202103030006368&gisu={gisu}&yminsa=2026"
|
|
"&searchData={year}0101{year}1231&color=#1C90FB"
|
|
"&companyName=(%EC%A3%BC)%ED%95%9C%EB%A7%A5%EA%B8%B0%EC%88%A0&companyID=b21344"
|
|
)
|
|
|
|
|
|
def log(message: str) -> None:
|
|
print(f"[hanmac-refresh] {datetime.now().isoformat(timespec='seconds')} {message}", flush=True)
|
|
|
|
|
|
def normalized_rows(path: Path, account: wehago.Account) -> list[tuple[str, ...]]:
|
|
rows = wehago.read_downloaded_rows(path)
|
|
_headers, ledger_rows = wehago.extract_ledger_data_rows(rows, account)
|
|
return [
|
|
tuple("" if value is None else str(value).strip() for value in values)
|
|
for _sheet_name, _row_number, values in ledger_rows
|
|
]
|
|
|
|
|
|
def row_digest(rows: list[tuple[str, ...]]) -> str:
|
|
digest = hashlib.sha256()
|
|
for row in rows:
|
|
digest.update(json.dumps(row, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
|
|
digest.update(b"\n")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def content_rows(rows: list[tuple[str, ...]]) -> list[tuple[str, ...]]:
|
|
"""계정코드/계정명 열을 제외해 서로 다른 계정의 동일 원장을 탐지합니다."""
|
|
return [row[:7] for row in rows]
|
|
|
|
|
|
def validate_staging(staging_dir: Path) -> dict[str, Any]:
|
|
accounts = wehago.discover_downloaded_accounts(staging_dir)
|
|
digest_accounts: dict[str, list[dict[str, str]]] = {}
|
|
failures: list[dict[str, str]] = []
|
|
|
|
for account in accounts:
|
|
path = wehago.find_account_file(account, staging_dir)
|
|
if path is None:
|
|
continue
|
|
try:
|
|
rows = normalized_rows(path, account)
|
|
except Exception as exc:
|
|
failures.append(
|
|
{"account_code": account.code, "account_name": account.name, "reason": f"파일 읽기 실패: {exc}"}
|
|
)
|
|
continue
|
|
if not rows:
|
|
continue
|
|
digest = row_digest(content_rows(rows))
|
|
digest_accounts.setdefault(digest, []).append(
|
|
{"account_code": account.code, "account_name": account.name, "path": str(path)}
|
|
)
|
|
|
|
duplicate_contents = [group for group in digest_accounts.values() if len(group) > 1]
|
|
progress_path = staging_dir / "_api_progress.json"
|
|
api_provenance: dict[str, Any] = {"passed": False, "reason": "API 진행 증빙 파일이 없습니다."}
|
|
if progress_path.exists():
|
|
try:
|
|
progress = json.loads(progress_path.read_text(encoding="utf-8"))
|
|
manifest_codes = {
|
|
str(item.get("account_code"))
|
|
for item in progress.get("accounts", [])
|
|
if item.get("account_code")
|
|
}
|
|
staging_codes = {account.code for account in accounts}
|
|
api_provenance = {
|
|
"passed": progress.get("status") in {"completed", "completed_with_failures"} and manifest_codes == staging_codes,
|
|
"status": progress.get("status"),
|
|
"manifest_accounts": len(manifest_codes),
|
|
"staging_accounts": len(staging_codes),
|
|
"download_failures": progress.get("failures", []),
|
|
"missing_manifest_codes": sorted(staging_codes - manifest_codes),
|
|
"missing_staging_codes": sorted(manifest_codes - staging_codes),
|
|
}
|
|
except (OSError, ValueError, TypeError) as exc:
|
|
api_provenance = {"passed": False, "reason": f"API 진행 증빙 읽기 실패: {exc}"}
|
|
downloaded_files_valid = not failures and not duplicate_contents and api_provenance["passed"]
|
|
return {
|
|
"passed": downloaded_files_valid,
|
|
"downloaded_files_valid": downloaded_files_valid,
|
|
"file_failures": failures,
|
|
"duplicate_contents": duplicate_contents,
|
|
"api_provenance": api_provenance,
|
|
}
|
|
|
|
|
|
def compare_and_promote(
|
|
year: int,
|
|
staging_dir: Path,
|
|
canonical_dir: Path,
|
|
backup_dir: Path,
|
|
*,
|
|
promote: bool = False,
|
|
allow_partial_promote: bool = False,
|
|
) -> dict[str, Any]:
|
|
staging_accounts = wehago.discover_downloaded_accounts(staging_dir)
|
|
canonical_accounts = {account.code: account for account in wehago.discover_downloaded_accounts(canonical_dir)}
|
|
validation = validate_staging(staging_dir)
|
|
staging_codes = {account.code for account in staging_accounts}
|
|
missing = [
|
|
{"account_code": code, "account_name": account.name}
|
|
for code, account in canonical_accounts.items()
|
|
if code not in staging_codes
|
|
]
|
|
validation["missing_accounts"] = missing
|
|
validation["passed"] = validation["passed"] and not missing
|
|
report: dict[str, Any] = {
|
|
"year": year,
|
|
"validation": validation,
|
|
"promoted": promote and validation["passed"],
|
|
"changed": [],
|
|
"unchanged": [],
|
|
"new": [],
|
|
"missing": missing,
|
|
}
|
|
should_promote = bool(report["promoted"]) or (
|
|
promote and allow_partial_promote and bool(validation.get("downloaded_files_valid"))
|
|
)
|
|
report["partial_promoted"] = should_promote and not report["promoted"]
|
|
|
|
for account in staging_accounts:
|
|
new_path = wehago.find_account_file(account, staging_dir)
|
|
old_account = canonical_accounts.get(account.code, account)
|
|
old_path = wehago.find_account_file(old_account, canonical_dir)
|
|
if new_path is None:
|
|
continue
|
|
|
|
new_rows = normalized_rows(new_path, account)
|
|
new_digest = row_digest(new_rows)
|
|
item = {
|
|
"account_code": account.code,
|
|
"account_name": account.name,
|
|
"new_rows": len(new_rows),
|
|
"new_digest": new_digest,
|
|
}
|
|
if old_path is None:
|
|
if should_promote:
|
|
target = canonical_dir / new_path.name
|
|
shutil.copy2(new_path, target)
|
|
report["new"].append(item)
|
|
continue
|
|
|
|
old_rows = normalized_rows(old_path, old_account)
|
|
old_digest = row_digest(old_rows)
|
|
item.update({"old_rows": len(old_rows), "old_digest": old_digest})
|
|
if old_digest == new_digest:
|
|
report["unchanged"].append(item)
|
|
continue
|
|
|
|
if should_promote:
|
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(old_path, backup_dir / old_path.name)
|
|
target = canonical_dir / new_path.name
|
|
if target != old_path and old_path.exists():
|
|
old_path.unlink()
|
|
shutil.copy2(new_path, target)
|
|
report["changed"].append(item)
|
|
|
|
return report
|
|
|
|
|
|
def import_years_to_db(merged_paths: dict[int, Path], run_root: Path) -> dict[str, Any]:
|
|
db_backup = run_root / f"data_before_refresh_{datetime.now():%Y%m%d_%H%M%S}.db"
|
|
shutil.copy2(DB_PATH, db_backup)
|
|
engine = create_engine(f"sqlite:///{DB_PATH}", connect_args={"check_same_thread": False})
|
|
imported: dict[str, Any] = {}
|
|
|
|
with engine.begin() as conn:
|
|
for year, path in merged_paths.items():
|
|
file_kind, header, sheet_name = detect_file_kind(path)
|
|
if file_kind != "ledger":
|
|
raise ValueError(f"통합 파일이 계정별원장 형식이 아닙니다: {path}")
|
|
|
|
conn.execute(text("DELETE FROM wehago_ledger_rows WHERE fiscal_year = :year"), {"year": year})
|
|
workbook = wehago.load_workbook(path, read_only=True, data_only=True)
|
|
try:
|
|
sheet = workbook.worksheets[0]
|
|
sample_rows = list(sheet.iter_rows(min_row=2, max_row=51, values_only=True))
|
|
year_hint = infer_year_hint(path, file_kind, sample_rows) or year
|
|
source_id, _changed = upsert_source_file(conn, path, file_kind, year_hint, sheet_name, header)
|
|
conn.execute(text("DELETE FROM wehago_ledger_rows WHERE source_file_id = :source_id"), {"source_id": source_id})
|
|
inserted = import_ledger_rows(conn, source_id, sheet_name, sheet.iter_rows(min_row=2, values_only=True), year)
|
|
conn.execute(
|
|
text(
|
|
"UPDATE wehago_source_files "
|
|
"SET row_count = :row_count, imported_at = CURRENT_TIMESTAMP WHERE id = :source_id"
|
|
),
|
|
{"row_count": inserted, "source_id": source_id},
|
|
)
|
|
imported[str(year)] = {"path": str(path), "rows": inserted, "source_id": source_id}
|
|
finally:
|
|
workbook.close()
|
|
|
|
conn.execute(
|
|
text(
|
|
"DELETE FROM wehago_source_files WHERE id NOT IN "
|
|
"(SELECT source_file_id FROM wehago_ledger_rows UNION SELECT source_file_id FROM wehago_voucher_rows)"
|
|
)
|
|
)
|
|
rebuild_comparison_results(conn)
|
|
for table in (
|
|
"wehago_metric_count_cache",
|
|
"wehago_result_row_cache",
|
|
"wehago_pair_recommend_cache",
|
|
"wehago_summary_range_cache",
|
|
):
|
|
conn.execute(text(f"DELETE FROM {table}"))
|
|
|
|
return {"db_backup": str(db_backup), "imported": imported}
|
|
|
|
|
|
def run_download(
|
|
year: int,
|
|
staging_dir: Path,
|
|
debugger_address: str,
|
|
download_mode: str = "api",
|
|
target_accounts: list[wehago.Account] | None = None,
|
|
) -> None:
|
|
wehago.CHROME_DEBUGGER_ADDRESS = debugger_address
|
|
wehago.DOWNLOAD_DIR = staging_dir
|
|
wehago.FORCE_REDOWNLOAD = True
|
|
wehago.SKIP_ALREADY_DOWNLOADED = False
|
|
wehago.SEQUENTIAL_ACCOUNT_MODE = False
|
|
wehago.SCAN_EXISTING_ACCOUNT_MODE = False
|
|
wehago.SCAN_ALL_VISIBLE_ACCOUNT_MODE = False
|
|
wehago.ALLOW_UNCHANGED_DETAIL = False
|
|
wehago.OPEN_ACCOUNT_LEDGER_URL = True
|
|
wehago.REFRESH_BEFORE_RUN = False
|
|
wehago.EXPECTED_PERIOD_START = f"{year}.01.01"
|
|
wehago.EXPECTED_PERIOD_END = f"{year}.12.31"
|
|
wehago.ACCOUNT_LEDGER_URL = HANMAC_LEDGER_URL.format(year=year, gisu=GISU_BY_YEAR[year])
|
|
wehago.MERGED_FILENAME = f"{year}_계정별원장_취합.xlsx"
|
|
if download_mode == "excel":
|
|
wehago.run(wehago.ACCOUNTS, merge=False, pause_on_failure=False)
|
|
return
|
|
|
|
driver = wehago.build_driver(staging_dir)
|
|
try:
|
|
wehago.open_wehago(driver)
|
|
wehago.ensure_ledger_data_loaded(driver, wehago.ACCOUNTS)
|
|
accounts = target_accounts or wehago.ACCOUNTS
|
|
if not accounts:
|
|
raise RuntimeError("API 다운로드 대상 계정이 없습니다.")
|
|
wehago_api.download_accounts(driver, accounts, staging_dir)
|
|
finally:
|
|
driver.quit()
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="한맥기술 WEHAGO 계정별원장 재다운로드, 비교, 취합 및 DB 반영")
|
|
parser.add_argument("--years", nargs="+", type=int, default=list(YEARS))
|
|
parser.add_argument("--debugger-address", default=os.environ.get("WEHAGO_CHROME_DEBUGGER_ADDRESS", "127.0.0.1:9225"))
|
|
parser.add_argument("--skip-download", action="store_true", help="이미 받은 staging 파일로 비교부터 실행")
|
|
parser.add_argument("--skip-db", action="store_true", help="파일 비교와 취합까지만 실행")
|
|
parser.add_argument("--staging-root", type=Path, help="--skip-download 시 사용할 연도별 staging 상위 폴더")
|
|
parser.add_argument(
|
|
"--download-mode",
|
|
choices=("api", "excel"),
|
|
default="api",
|
|
help="기본값 api는 로그인 브라우저의 원장 API 응답을 검증하여 저장합니다.",
|
|
)
|
|
parser.add_argument(
|
|
"--promote-validated",
|
|
action="store_true",
|
|
help="검증을 통과한 staging만 기존 원장에 반영합니다. 기본값은 비교 보고서만 생성합니다.",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-partial-promote",
|
|
action="store_true",
|
|
help="API 검증을 통과한 계정 파일만 반영하고, 실패/누락 계정은 기존 파일을 유지합니다.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
invalid = sorted(set(args.years) - set(YEARS))
|
|
if invalid:
|
|
raise ValueError(f"지원하지 않는 연도입니다: {invalid}")
|
|
|
|
run_stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
run_root = WEHAGO_SOURCE_ROOT / "data_download" / "hanmac_refresh" / run_stamp
|
|
staging_root = args.staging_root or (run_root / "staging")
|
|
report_path = run_root / "refresh_report.json"
|
|
merged_paths: dict[int, Path] = {}
|
|
reports: list[dict[str, Any]] = []
|
|
|
|
for year in args.years:
|
|
canonical_dir = WEHAGO_SOURCE_ROOT / "data_download" / "hanmac" / str(year)
|
|
staging_dir = staging_root / str(year)
|
|
backup_dir = run_root / "replaced_originals" / str(year)
|
|
canonical_dir.mkdir(parents=True, exist_ok=True)
|
|
staging_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
if not args.skip_download:
|
|
log(f"{year}년 신규 원장 다운로드 시작: {staging_dir}")
|
|
try:
|
|
target_accounts = wehago.discover_downloaded_accounts(canonical_dir) or None
|
|
run_download(year, staging_dir, args.debugger_address, args.download_mode, target_accounts)
|
|
except RuntimeError as exc:
|
|
if not wehago.discover_downloaded_accounts(staging_dir):
|
|
raise
|
|
log(f"{year}년 일부 계정 다운로드 실패를 기존 파일 유지 방식으로 처리합니다: {exc}")
|
|
|
|
log(f"{year}년 기존 파일과 신규 파일 비교")
|
|
report = compare_and_promote(
|
|
year,
|
|
staging_dir,
|
|
canonical_dir,
|
|
backup_dir,
|
|
promote=args.promote_validated,
|
|
allow_partial_promote=args.allow_partial_promote,
|
|
)
|
|
reports.append(report)
|
|
log(
|
|
f"{year}년 비교 완료: 변경 {len(report['changed'])}, 신규 {len(report['new'])}, "
|
|
f"동일 {len(report['unchanged'])}, 신규 다운로드 누락 {len(report['missing'])}"
|
|
)
|
|
if not report["validation"]["passed"]:
|
|
log(
|
|
f"{year}년 검증 실패: 파일 오류 {len(report['validation']['file_failures'])}, "
|
|
f"서로 다른 계정의 동일 원장 {len(report['validation']['duplicate_contents'])}그룹"
|
|
)
|
|
elif not args.promote_validated:
|
|
log(f"{year}년 검증 통과. --promote-validated가 없어 기존 원장은 변경하지 않습니다.")
|
|
merged = wehago.consolidate_download_dir(canonical_dir, output_name=f"{year}_계정별원장_취합.xlsx")
|
|
merged_paths[year] = merged
|
|
log(f"{year}년 통합본 생성: {merged}")
|
|
|
|
payload: dict[str, Any] = {"run_stamp": run_stamp, "years": args.years, "reports": reports}
|
|
all_valid = all(report["validation"]["passed"] for report in reports)
|
|
all_promoted = all(report["promoted"] for report in reports)
|
|
if not args.skip_db and all_valid and all_promoted:
|
|
log("DB 백업 및 연도별 원장 교체 시작")
|
|
payload["db"] = import_years_to_db(merged_paths, run_root)
|
|
log("DB 반영 완료")
|
|
elif not args.skip_db:
|
|
payload["db_skipped"] = "모든 연도의 검증 통과 및 --promote-validated 지정 전에는 DB를 반영하지 않습니다."
|
|
log(payload["db_skipped"])
|
|
|
|
run_root.mkdir(parents=True, exist_ok=True)
|
|
report_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
log(f"작업 보고서: {report_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|