103 lines
3.8 KiB
Python
103 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
import scripts.refresh_hanmac_wehago_ledgers as refresh
|
|
import scripts.retry_failed_wehago_accounts_direct as direct
|
|
import scripts.wehago_data_download_2022_work as wehago
|
|
import scripts.wehago_ledger_api_download as api_download
|
|
|
|
|
|
BASE = Path(r"\\wsl.localhost\Ubuntu\home\b17301\WEHAGO_DB\data_download")
|
|
|
|
|
|
def main() -> int:
|
|
run_root = BASE / "hanmac_refresh" / f"verify_all_direct_{datetime.now():%Y%m%d_%H%M%S}"
|
|
run_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
wehago.CHROME_DEBUGGER_ADDRESS = "127.0.0.1:9225"
|
|
wehago.DOWNLOAD_DIR = run_root
|
|
driver = wehago.build_driver(run_root)
|
|
try:
|
|
cookies = direct.cookie_map(driver)
|
|
finally:
|
|
driver.quit()
|
|
|
|
reports: dict[str, object] = {}
|
|
for year in refresh.YEARS:
|
|
canonical = BASE / "hanmac" / str(year)
|
|
staging = run_root / "staging" / str(year)
|
|
staging.mkdir(parents=True, exist_ok=True)
|
|
accounts = wehago.discover_downloaded_accounts(canonical)
|
|
manifest: list[dict[str, object]] = []
|
|
failures: list[dict[str, str]] = []
|
|
|
|
print(f"YEAR {year}: verify all {len(accounts)} accounts", flush=True)
|
|
for index, account in enumerate(accounts, start=1):
|
|
try:
|
|
rows = direct.fetch_ledger_rows(year, account, cookies)
|
|
api_download.validate_api_rows(account, rows)
|
|
api_download.write_api_rows(staging / account.safe_filename, account, rows)
|
|
manifest.append(
|
|
{
|
|
"account_code": account.code,
|
|
"account_name": account.name,
|
|
"api_request_code": f"{account.code}00",
|
|
"api_response_rows": len(rows),
|
|
}
|
|
)
|
|
print(f"[{year} {index}/{len(accounts)}] OK {account.code} rows={len(rows)}", flush=True)
|
|
except Exception as exc:
|
|
reason = f"{type(exc).__name__}: {exc}"
|
|
failures.append({"account_code": account.code, "account_name": account.name, "reason": reason})
|
|
print(f"[{year} {index}/{len(accounts)}] FAIL {account.code}: {reason}", flush=True)
|
|
|
|
api_download.write_progress(
|
|
staging,
|
|
{
|
|
"status": "completed_with_failures" if failures else "completed",
|
|
"completed": len(manifest),
|
|
"total": len(accounts),
|
|
"accounts": manifest,
|
|
"failures": failures,
|
|
},
|
|
)
|
|
comparison = refresh.compare_and_promote(
|
|
year,
|
|
staging,
|
|
canonical,
|
|
run_root / "unused_backup" / str(year),
|
|
promote=False,
|
|
)
|
|
reports[str(year)] = {
|
|
"requested": len(accounts),
|
|
"downloaded": len(manifest),
|
|
"failures": failures,
|
|
"validation": comparison["validation"],
|
|
"changed": comparison["changed"],
|
|
"unchanged": comparison["unchanged"],
|
|
"new": comparison["new"],
|
|
"missing": comparison["missing"],
|
|
}
|
|
(run_root / "verification_report.json").write_text(
|
|
json.dumps({"reports": reports}, ensure_ascii=False, indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
print(
|
|
f"YEAR {year}: changed={len(comparison['changed'])} unchanged={len(comparison['unchanged'])} "
|
|
f"failures={len(failures)} missing={len(comparison['missing'])}",
|
|
flush=True,
|
|
)
|
|
|
|
print(f"REPORT {run_root / 'verification_report.json'}", flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|