from __future__ import annotations import json import sys from datetime import datetime from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import scripts.refresh_hanmac_wehago_ledgers as refresh import scripts.retry_failed_wehago_accounts_direct as direct import scripts.wehago_data_download_2022_work as wehago import scripts.wehago_ledger_api_download as api_download BASE = Path(r"\\wsl.localhost\Ubuntu\home\b17301\WEHAGO_DB\data_download") def main() -> int: run_root = BASE / "hanmac_refresh" / f"verify_all_direct_{datetime.now():%Y%m%d_%H%M%S}" run_root.mkdir(parents=True, exist_ok=True) wehago.CHROME_DEBUGGER_ADDRESS = "127.0.0.1:9225" wehago.DOWNLOAD_DIR = run_root driver = wehago.build_driver(run_root) try: cookies = direct.cookie_map(driver) finally: driver.quit() reports: dict[str, object] = {} for year in refresh.YEARS: canonical = BASE / "hanmac" / str(year) staging = run_root / "staging" / str(year) staging.mkdir(parents=True, exist_ok=True) accounts = wehago.discover_downloaded_accounts(canonical) manifest: list[dict[str, object]] = [] failures: list[dict[str, str]] = [] print(f"YEAR {year}: verify all {len(accounts)} accounts", flush=True) for index, account in enumerate(accounts, start=1): try: rows = direct.fetch_ledger_rows(year, account, cookies) api_download.validate_api_rows(account, rows) api_download.write_api_rows(staging / account.safe_filename, account, rows) manifest.append( { "account_code": account.code, "account_name": account.name, "api_request_code": f"{account.code}00", "api_response_rows": len(rows), } ) print(f"[{year} {index}/{len(accounts)}] OK {account.code} rows={len(rows)}", flush=True) except Exception as exc: reason = f"{type(exc).__name__}: {exc}" failures.append({"account_code": account.code, "account_name": account.name, "reason": reason}) print(f"[{year} {index}/{len(accounts)}] FAIL {account.code}: {reason}", flush=True) api_download.write_progress( staging, { "status": "completed_with_failures" if failures else "completed", "completed": len(manifest), "total": len(accounts), "accounts": manifest, "failures": failures, }, ) comparison = refresh.compare_and_promote( year, staging, canonical, run_root / "unused_backup" / str(year), promote=False, ) reports[str(year)] = { "requested": len(accounts), "downloaded": len(manifest), "failures": failures, "validation": comparison["validation"], "changed": comparison["changed"], "unchanged": comparison["unchanged"], "new": comparison["new"], "missing": comparison["missing"], } (run_root / "verification_report.json").write_text( json.dumps({"reports": reports}, ensure_ascii=False, indent=2), encoding="utf-8", ) print( f"YEAR {year}: changed={len(comparison['changed'])} unchanged={len(comparison['unchanged'])} " f"failures={len(failures)} missing={len(comparison['missing'])}", flush=True, ) print(f"REPORT {run_root / 'verification_report.json'}", flush=True) return 0 if __name__ == "__main__": raise SystemExit(main())