93 lines
3.2 KiB
Python
93 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
import scripts.refresh_hanmac_wehago_ledgers as refresh
|
|
import scripts.wehago_data_download_2022_work as wehago
|
|
|
|
|
|
SOURCE_RUNS = {
|
|
2022: "20260615_194953",
|
|
2023: "20260615_195024",
|
|
2024: "20260615_195215",
|
|
2025: "20260615_195405",
|
|
}
|
|
|
|
|
|
def wsl_unc(path: str) -> Path:
|
|
if os.name == "nt" and path.startswith("/home/"):
|
|
return Path(r"\\wsl.localhost\Ubuntu" + path.replace("/", "\\"))
|
|
return Path(path)
|
|
|
|
|
|
def report_entries(report: dict) -> list[dict]:
|
|
reports = report.get("reports")
|
|
if isinstance(reports, list):
|
|
return reports
|
|
if isinstance(reports, dict):
|
|
return list(reports.values())
|
|
return [report]
|
|
|
|
|
|
def failed_codes_from_report(report_path: Path) -> list[str]:
|
|
report = json.loads(report_path.read_text(encoding="utf-8"))
|
|
entries = report_entries(report)
|
|
if not entries:
|
|
return []
|
|
validation = entries[0].get("validation", {})
|
|
api = validation.get("api_provenance") or validation.get("api_progress") or {}
|
|
return [
|
|
str(item["account_code"])
|
|
for item in api.get("download_failures", [])
|
|
if item.get("account_code")
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
base = wsl_unc("/home/b17301/WEHAGO_DB/data_download/hanmac_refresh")
|
|
canonical_base = wsl_unc("/home/b17301/WEHAGO_DB/data_download/hanmac")
|
|
run_root = base / f"retry_failed_{datetime.now():%Y%m%d_%H%M%S}"
|
|
run_root.mkdir(parents=True, exist_ok=True)
|
|
summary: dict[str, object] = {}
|
|
|
|
for year, source_run in SOURCE_RUNS.items():
|
|
failed_codes = failed_codes_from_report(base / source_run / "refresh_report.json")
|
|
canonical_dir = canonical_base / str(year)
|
|
by_code = {account.code: account for account in wehago.discover_downloaded_accounts(canonical_dir)}
|
|
accounts = [by_code[code] for code in failed_codes if code in by_code]
|
|
staging_dir = run_root / "staging" / str(year)
|
|
staging_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
print(f"YEAR {year}: retry accounts={len(accounts)} staging={staging_dir}", flush=True)
|
|
refresh.run_download(year, staging_dir, "127.0.0.1:9225", "api", accounts)
|
|
validation = refresh.validate_staging(staging_dir)
|
|
api = validation.get("api_provenance", {})
|
|
failures = api.get("download_failures", [])
|
|
summary[str(year)] = {
|
|
"requested": len(accounts),
|
|
"downloaded": api.get("manifest_accounts"),
|
|
"failures": len(failures),
|
|
"failed_codes": [str(item.get("account_code")) for item in failures],
|
|
"staging": str(staging_dir),
|
|
"validation": validation,
|
|
}
|
|
(run_root / "retry_summary.json").write_text(
|
|
json.dumps(summary, ensure_ascii=False, indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
print(f"YEAR {year}: downloaded={api.get('manifest_accounts')} failures={len(failures)}", flush=True)
|
|
|
|
print(f"SUMMARY_PATH {run_root / 'retry_summary.json'}", flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|