from __future__ import annotations import json import os import sys from datetime import datetime from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) import scripts.refresh_hanmac_wehago_ledgers as refresh import scripts.wehago_data_download_2022_work as wehago SOURCE_RUNS = { 2022: "20260615_194953", 2023: "20260615_195024", 2024: "20260615_195215", 2025: "20260615_195405", } def wsl_unc(path: str) -> Path: if os.name == "nt" and path.startswith("/home/"): return Path(r"\\wsl.localhost\Ubuntu" + path.replace("/", "\\")) return Path(path) def report_entries(report: dict) -> list[dict]: reports = report.get("reports") if isinstance(reports, list): return reports if isinstance(reports, dict): return list(reports.values()) return [report] def failed_codes_from_report(report_path: Path) -> list[str]: report = json.loads(report_path.read_text(encoding="utf-8")) entries = report_entries(report) if not entries: return [] validation = entries[0].get("validation", {}) api = validation.get("api_provenance") or validation.get("api_progress") or {} return [ str(item["account_code"]) for item in api.get("download_failures", []) if item.get("account_code") ] def main() -> int: base = wsl_unc("/home/b17301/WEHAGO_DB/data_download/hanmac_refresh") canonical_base = wsl_unc("/home/b17301/WEHAGO_DB/data_download/hanmac") run_root = base / f"retry_failed_{datetime.now():%Y%m%d_%H%M%S}" run_root.mkdir(parents=True, exist_ok=True) summary: dict[str, object] = {} for year, source_run in SOURCE_RUNS.items(): failed_codes = failed_codes_from_report(base / source_run / "refresh_report.json") canonical_dir = canonical_base / str(year) by_code = {account.code: account for account in wehago.discover_downloaded_accounts(canonical_dir)} accounts = [by_code[code] for code in failed_codes if code in by_code] staging_dir = run_root / "staging" / str(year) staging_dir.mkdir(parents=True, exist_ok=True) print(f"YEAR {year}: retry accounts={len(accounts)} staging={staging_dir}", flush=True) refresh.run_download(year, staging_dir, "127.0.0.1:9225", "api", accounts) validation = refresh.validate_staging(staging_dir) api = validation.get("api_provenance", {}) failures = api.get("download_failures", []) summary[str(year)] = { "requested": len(accounts), "downloaded": api.get("manifest_accounts"), "failures": len(failures), "failed_codes": [str(item.get("account_code")) for item in failures], "staging": str(staging_dir), "validation": validation, } (run_root / "retry_summary.json").write_text( json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8", ) print(f"YEAR {year}: downloaded={api.get('manifest_accounts')} failures={len(failures)}", flush=True) print(f"SUMMARY_PATH {run_root / 'retry_summary.json'}", flush=True) return 0 if __name__ == "__main__": raise SystemExit(main())