Update wehago matching logic and exclude reports

2026-07-03 09:07:04 +09:00
parent a09190ecd3
commit da073ad6ef
62 changed files with 36940 additions and 1554 deletions
@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import scripts.refresh_hanmac_wehago_ledgers as refresh
+import scripts.retry_failed_wehago_accounts_direct as direct
+import scripts.wehago_data_download_2022_work as wehago
+import scripts.wehago_ledger_api_download as api_download
+
+
+BASE = Path(r"\\wsl.localhost\Ubuntu\home\b17301\WEHAGO_DB\data_download")
+
+
+def main() -> int:
+    run_root = BASE / "hanmac_refresh" / f"verify_all_direct_{datetime.now():%Y%m%d_%H%M%S}"
+    run_root.mkdir(parents=True, exist_ok=True)
+
+    wehago.CHROME_DEBUGGER_ADDRESS = "127.0.0.1:9225"
+    wehago.DOWNLOAD_DIR = run_root
+    driver = wehago.build_driver(run_root)
+    try:
+        cookies = direct.cookie_map(driver)
+    finally:
+        driver.quit()
+
+    reports: dict[str, object] = {}
+    for year in refresh.YEARS:
+        canonical = BASE / "hanmac" / str(year)
+        staging = run_root / "staging" / str(year)
+        staging.mkdir(parents=True, exist_ok=True)
+        accounts = wehago.discover_downloaded_accounts(canonical)
+        manifest: list[dict[str, object]] = []
+        failures: list[dict[str, str]] = []
+
+        print(f"YEAR {year}: verify all {len(accounts)} accounts", flush=True)
+        for index, account in enumerate(accounts, start=1):
+            try:
+                rows = direct.fetch_ledger_rows(year, account, cookies)
+                api_download.validate_api_rows(account, rows)
+                api_download.write_api_rows(staging / account.safe_filename, account, rows)
+                manifest.append(
+                    {
+                        "account_code": account.code,
+                        "account_name": account.name,
+                        "api_request_code": f"{account.code}00",
+                        "api_response_rows": len(rows),
+                    }
+                )
+                print(f"[{year} {index}/{len(accounts)}] OK {account.code} rows={len(rows)}", flush=True)
+            except Exception as exc:
+                reason = f"{type(exc).__name__}: {exc}"
+                failures.append({"account_code": account.code, "account_name": account.name, "reason": reason})
+                print(f"[{year} {index}/{len(accounts)}] FAIL {account.code}: {reason}", flush=True)
+
+        api_download.write_progress(
+            staging,
+            {
+                "status": "completed_with_failures" if failures else "completed",
+                "completed": len(manifest),
+                "total": len(accounts),
+                "accounts": manifest,
+                "failures": failures,
+            },
+        )
+        comparison = refresh.compare_and_promote(
+            year,
+            staging,
+            canonical,
+            run_root / "unused_backup" / str(year),
+            promote=False,
+        )
+        reports[str(year)] = {
+            "requested": len(accounts),
+            "downloaded": len(manifest),
+            "failures": failures,
+            "validation": comparison["validation"],
+            "changed": comparison["changed"],
+            "unchanged": comparison["unchanged"],
+            "new": comparison["new"],
+            "missing": comparison["missing"],
+        }
+        (run_root / "verification_report.json").write_text(
+            json.dumps({"reports": reports}, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        print(
+            f"YEAR {year}: changed={len(comparison['changed'])} unchanged={len(comparison['unchanged'])} "
+            f"failures={len(failures)} missing={len(comparison['missing'])}",
+            flush=True,
+        )
+
+    print(f"REPORT {run_root / 'verification_report.json'}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())