from __future__ import annotations import argparse import json import sqlite3 import sys from datetime import datetime from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from runtime_config import ( # noqa: E402 BACKUP_DIR, DB_PATH, WAL_BLOCK_HEAVY_BYTES, WAL_WARN_BYTES, ensure_runtime_directories, sqlite_runtime_status, ) CACHE_TABLES = ( "wehago_compare_export_row_cache", "wehago_compare_query_page_cache", "wehago_compare_query_groups", "wehago_compare_query_rows", "wehago_metric_count_cache", "wehago_pair_recommend_cache", "wehago_raw_erp_trace_candidate_cache", "wehago_result_row_cache", "wehago_summary_range_cache", ) SOURCE_TABLES = ("wehago_voucher_rows", "wehago_ledger_rows") def _size(path: Path) -> int: return path.stat().st_size if path.exists() else 0 def _table_counts(conn: sqlite3.Connection, tables: tuple[str, ...]) -> dict[str, int | None]: existing = { str(row[0]) for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall() } counts: dict[str, int | None] = {} for table in tables: if table not in existing: counts[table] = None continue counts[table] = int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) return counts def _table_storage(conn: sqlite3.Connection, tables: tuple[str, ...]) -> dict[str, dict[str, int | None]]: existing = { str(row[0]) for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall() } has_dbstat = True try: conn.execute("SELECT 1 FROM dbstat LIMIT 1").fetchone() except sqlite3.DatabaseError: has_dbstat = False page_size = int(conn.execute("PRAGMA page_size").fetchone()[0]) storage: dict[str, dict[str, int | None]] = {} for table in tables: if table not in existing: storage[table] = {"rows": None, "bytes": None} continue rows = int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) bytes_used: int | None = None if has_dbstat: try: stat = conn.execute( "SELECT SUM(pgsize) FROM dbstat WHERE name = ?", (table,), ).fetchone() bytes_used = int(stat[0] or 0) except sqlite3.DatabaseError: bytes_used = None storage[table] = {"rows": rows, "bytes": bytes_used if has_dbstat else None} if not has_dbstat: for table in storage.values(): table["estimated_page_size"] = page_size return storage def _table_exists(conn: sqlite3.Connection, table: str) -> bool: return bool( conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (table,), ).fetchone() ) def cache_retention_report(limit: int) -> dict[str, Any]: conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30) try: if not _table_exists(conn, "wehago_compare_export_row_cache"): return {"export_row_cache": {"exists": False}} signatures = [ { "fiscal_year": int(row[0]), "snapshot_signature": str(row[1]), "rows": int(row[2]), } for row in conn.execute( """ SELECT fiscal_year, snapshot_signature, COUNT(*) AS rows FROM wehago_compare_export_row_cache GROUP BY fiscal_year, snapshot_signature ORDER BY fiscal_year DESC, rows DESC LIMIT ? """, (limit,), ).fetchall() ] orphan_rows = 0 if _table_exists(conn, "wehago_snapshot_status"): orphan_rows = int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_export_row_cache c LEFT JOIN wehago_snapshot_status s ON s.fiscal_year = c.fiscal_year AND s.snapshot_signature = c.snapshot_signature AND s.state = 'ready' WHERE s.fiscal_year IS NULL """ ).fetchone()[0] ) return { "export_row_cache": { "exists": True, "signature_groups_sample": signatures, "orphan_rows_not_matching_ready_snapshot": orphan_rows, } } finally: conn.close() def prune_orphan_export_cache(dry_run: bool, acknowledged: bool) -> dict[str, Any]: if not dry_run and not acknowledged: raise RuntimeError( "실제 삭제는 `--ack-delete-rebuildable-cache`를 함께 지정해야 합니다. " "삭제 대상은 현재 ready 스냅샷 서명과 맞지 않는 재생성 가능 export-row 캐시입니다." ) conn = sqlite3.connect(DB_PATH, timeout=30) try: if not _table_exists(conn, "wehago_compare_export_row_cache") or not _table_exists( conn, "wehago_snapshot_status", ): return {"dry_run": dry_run, "candidate_rows": 0, "deleted_rows": 0} candidate_rows = int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_export_row_cache c LEFT JOIN wehago_snapshot_status s ON s.fiscal_year = c.fiscal_year AND s.snapshot_signature = c.snapshot_signature AND s.state = 'ready' WHERE s.fiscal_year IS NULL """ ).fetchone()[0] ) deleted_rows = 0 if not dry_run and candidate_rows: conn.execute( """ DELETE FROM wehago_compare_export_row_cache WHERE NOT EXISTS ( SELECT 1 FROM wehago_snapshot_status s WHERE s.fiscal_year = wehago_compare_export_row_cache.fiscal_year AND s.snapshot_signature = wehago_compare_export_row_cache.snapshot_signature AND s.state = 'ready' ) """ ) deleted_rows = conn.total_changes conn.commit() return {"dry_run": dry_run, "candidate_rows": candidate_rows, "deleted_rows": deleted_rows} finally: conn.close() def query_projection_retention_report(keep: int) -> dict[str, Any]: conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30) conn.row_factory = sqlite3.Row try: if not _table_exists(conn, "wehago_compare_query_groups"): return {"query_projection_cache": {"exists": False}} groups = conn.execute( """ SELECT start_year, end_year, signature, COUNT(*) AS group_rows, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups GROUP BY start_year, end_year, signature ORDER BY start_year DESC, end_year DESC, max_updated_at DESC """ ).fetchall() keep = max(1, int(keep)) by_scope: dict[tuple[int, int], list[sqlite3.Row]] = {} for row in groups: by_scope.setdefault((int(row["start_year"]), int(row["end_year"])), []).append(row) obsolete: list[sqlite3.Row] = [] retained: list[dict[str, Any]] = [] for scope, rows in by_scope.items(): for idx, row in enumerate(rows): item = { "start_year": int(row["start_year"]), "end_year": int(row["end_year"]), "signature": str(row["signature"] or ""), "group_rows": int(row["group_rows"] or 0), "max_updated_at": str(row["max_updated_at"] or ""), "retained": idx < keep, } if idx < keep: retained.append(item) else: obsolete.append(row) obsolete_group_rows = sum(int(row["group_rows"] or 0) for row in obsolete) obsolete_query_rows = 0 obsolete_page_rows = 0 for row in obsolete: params = ( int(row["start_year"]), int(row["end_year"]), str(row["signature"] or ""), ) if _table_exists(conn, "wehago_compare_query_rows"): obsolete_query_rows += int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_query_rows WHERE start_year = ? AND end_year = ? AND signature = ? """, params, ).fetchone()[0] ) if _table_exists(conn, "wehago_compare_query_page_cache"): obsolete_page_rows += int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_query_page_cache WHERE start_year = ? AND end_year = ? AND signature = ? """, params, ).fetchone()[0] ) return { "query_projection_cache": { "exists": True, "keep_per_range": keep, "range_signature_count": len(groups), "obsolete_signature_count": len(obsolete), "obsolete_group_rows": obsolete_group_rows, "obsolete_query_rows": obsolete_query_rows, "obsolete_page_rows": obsolete_page_rows, "retained_sample": retained[:20], } } finally: conn.close() def prune_old_query_projections(dry_run: bool, acknowledged: bool, keep: int) -> dict[str, Any]: if not dry_run and not acknowledged: raise RuntimeError( "실제 삭제는 `--ack-delete-rebuildable-cache`를 함께 지정해야 합니다. " "삭제 대상은 범위별 최신 N개를 제외한 재생성 가능 query projection 캐시입니다." ) conn = sqlite3.connect(DB_PATH, timeout=30) conn.row_factory = sqlite3.Row try: if not _table_exists(conn, "wehago_compare_query_groups"): return {"dry_run": dry_run, "candidate_group_rows": 0, "candidate_query_rows": 0, "candidate_page_rows": 0} keep = max(1, int(keep)) rows = conn.execute( """ SELECT start_year, end_year, signature, MAX(updated_at) AS max_updated_at FROM wehago_compare_query_groups GROUP BY start_year, end_year, signature ORDER BY start_year, end_year, max_updated_at DESC """ ).fetchall() by_scope: dict[tuple[int, int], list[sqlite3.Row]] = {} for row in rows: by_scope.setdefault((int(row["start_year"]), int(row["end_year"])), []).append(row) obsolete = [row for scope_rows in by_scope.values() for row in scope_rows[keep:]] candidate_group_rows = 0 candidate_query_rows = 0 candidate_page_rows = 0 deleted_rows = 0 for row in obsolete: params = ( int(row["start_year"]), int(row["end_year"]), str(row["signature"] or ""), ) candidate_group_rows += int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ? """, params, ).fetchone()[0] ) if _table_exists(conn, "wehago_compare_query_rows"): candidate_query_rows += int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_query_rows WHERE start_year = ? AND end_year = ? AND signature = ? """, params, ).fetchone()[0] ) if _table_exists(conn, "wehago_compare_query_page_cache"): candidate_page_rows += int( conn.execute( """ SELECT COUNT(*) FROM wehago_compare_query_page_cache WHERE start_year = ? AND end_year = ? AND signature = ? """, params, ).fetchone()[0] ) if not dry_run: for table in ( "wehago_compare_query_page_cache", "wehago_compare_query_rows", "wehago_compare_query_groups", ): if not _table_exists(conn, table): continue before = conn.total_changes conn.execute( f""" DELETE FROM {table} WHERE start_year = ? AND end_year = ? AND signature = ? """, params, ) deleted_rows += conn.total_changes - before conn.commit() return { "dry_run": dry_run, "keep_per_range": keep, "obsolete_signature_count": len(obsolete), "candidate_group_rows": candidate_group_rows, "candidate_query_rows": candidate_query_rows, "candidate_page_rows": candidate_page_rows, "deleted_rows": deleted_rows, } finally: conn.close() def status(include_counts: bool) -> dict[str, Any]: path = DB_PATH payload: dict[str, Any] = { "database_path": str(path), "database_exists": path.exists(), "files": { "database_bytes": _size(path), "wal_bytes": _size(path.with_name(path.name + "-wal")), "shm_bytes": _size(path.with_name(path.name + "-shm")), }, "runtime": sqlite_runtime_status(), "thresholds": { "wal_warn_bytes": WAL_WARN_BYTES, "wal_block_heavy_bytes": WAL_BLOCK_HEAVY_BYTES, }, } if not path.exists(): return payload conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5) try: payload["database"] = { "journal_mode": str(conn.execute("PRAGMA journal_mode").fetchone()[0]), "page_size": int(conn.execute("PRAGMA page_size").fetchone()[0]), "page_count": int(conn.execute("PRAGMA page_count").fetchone()[0]), "freelist_count": int(conn.execute("PRAGMA freelist_count").fetchone()[0]), "wal_autocheckpoint": int(conn.execute("PRAGMA wal_autocheckpoint").fetchone()[0]), } if include_counts: payload["source_table_counts"] = _table_counts(conn, SOURCE_TABLES) payload["cache_table_counts"] = _table_counts(conn, CACHE_TABLES) payload["cache_table_storage"] = _table_storage(conn, CACHE_TABLES) finally: conn.close() wal_bytes = int(payload["files"]["wal_bytes"]) warnings: list[str] = [] if not payload["runtime"]["meets_minimum_safe_version"]: warnings.append( "SQLite 런타임이 3.51.3 미만입니다. WAL DB를 운영하기 전에 안전 런타임으로 교체하세요." ) if wal_bytes >= WAL_BLOCK_HEAVY_BYTES: warnings.append("WAL이 중단 기준을 넘었습니다. 신규 대량 캐시 재생성을 보류하고 유지보수 창을 확보하세요.") elif wal_bytes >= WAL_WARN_BYTES: warnings.append("WAL이 경고 기준을 넘었습니다. 긴 조회/쓰기 작업과 checkpoint 상태를 점검하세요.") payload["warnings"] = warnings return payload def backup(output: Path | None, verify: str) -> dict[str, Any]: if not DB_PATH.exists(): raise FileNotFoundError(DB_PATH) ensure_runtime_directories() output = output or BACKUP_DIR / f"data-migration-{datetime.now():%Y%m%d-%H%M%S}.sqlite3" output.parent.mkdir(parents=True, exist_ok=True) temp_path = output.with_name("." + output.name + ".tmp") source_conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30) target_conn = sqlite3.connect(temp_path) last_reported_percent = -10 def report_progress(_status: int, remaining: int, total: int) -> None: nonlocal last_reported_percent percent = int((total - remaining) * 100 / total) if total else 100 reported_percent = min(100, (percent // 10) * 10) if reported_percent > last_reported_percent: print(f"backup copy progress: {reported_percent}%", file=sys.stderr, flush=True) last_reported_percent = reported_percent try: source_conn.backup(target_conn, pages=8192, progress=report_progress) check = "skipped" if verify == "smoke": target_conn.execute("PRAGMA schema_version").fetchone() target_conn.execute("SELECT COUNT(*) FROM sqlite_master").fetchone() check = "open-and-schema-readable" elif verify != "none": pragma = "integrity_check" if verify == "full" else "quick_check" print(f"backup verification started: {pragma}", file=sys.stderr, flush=True) check = str(target_conn.execute(f"PRAGMA {pragma}").fetchone()[0]) if check.lower() != "ok": raise RuntimeError(f"백업 {pragma} 실패: {check}") finally: target_conn.close() source_conn.close() temp_path.replace(output) return {"backup_path": str(output), "backup_bytes": _size(output), "verification": verify, "check_result": check} def checkpoint(mode: str, acknowledged: bool) -> dict[str, Any]: if not acknowledged: raise RuntimeError( "checkpoint는 운영 서버와 대량 작업을 중단한 유지보수 창에서만 실행하세요. " "`--ack-maintenance-window`를 함께 지정해야 합니다." ) conn = sqlite3.connect(DB_PATH, timeout=30) try: before = _size(DB_PATH.with_name(DB_PATH.name + "-wal")) result = conn.execute(f"PRAGMA wal_checkpoint({mode.upper()})").fetchone() after = _size(DB_PATH.with_name(DB_PATH.name + "-wal")) finally: conn.close() return {"mode": mode, "checkpoint_result": list(result or ()), "wal_bytes_before": before, "wal_bytes_after": after} def prepare_runtime_layout(root: Path) -> dict[str, str]: paths = { "db": root / "db", "cache": root / "cache", "backups": root / "backups", "exports": root / "exports", } for path in paths.values(): path.mkdir(parents=True, exist_ok=True) env_file = root / "runtime.env.example" env_file.write_text( "\n".join( [ f"INTRANET_DB_PATH={paths['db'] / 'data.db'}", f"INTRANET_BACKUP_DIR={paths['backups']}", f"INTRANET_CACHE_ROOT={paths['cache']}", f"INTRANET_COMPARE_EXPORT_DIR={paths['exports'] / 'wehago_compare'}", "INTRANET_REQUIRE_SAFE_SQLITE=1", f"WEHAGO_SOURCE_ROOT={Path.home() / 'WEHAGO_DB'}", "", ] ), encoding="utf-8", ) return {key: str(path) for key, path in paths.items()} | {"env_example": str(env_file)} def main() -> None: parser = argparse.ArgumentParser(description="SQLite/WAL runtime inspection and maintenance helpers.") subparsers = parser.add_subparsers(dest="command", required=True) status_parser = subparsers.add_parser("status") status_parser.add_argument("--include-counts", action="store_true") backup_parser = subparsers.add_parser("backup") backup_parser.add_argument("--output", type=Path) backup_parser.add_argument( "--verify", choices=("smoke", "quick", "full", "none"), default="smoke", help="smoke is suitable for container trials; use quick/full during a maintenance window.", ) checkpoint_parser = subparsers.add_parser("checkpoint") checkpoint_parser.add_argument("--mode", choices=("passive", "full", "restart", "truncate"), default="passive") checkpoint_parser.add_argument("--ack-maintenance-window", action="store_true") layout_parser = subparsers.add_parser("prepare-layout") layout_parser.add_argument("--root", type=Path, default=Path.home() / "intranet-runtime") report_parser = subparsers.add_parser("cache-retention-report") report_parser.add_argument("--limit", type=int, default=20) prune_parser = subparsers.add_parser("prune-orphan-export-cache") prune_parser.add_argument("--execute", action="store_true") prune_parser.add_argument("--ack-delete-rebuildable-cache", action="store_true") query_report_parser = subparsers.add_parser("query-retention-report") query_report_parser.add_argument("--keep", type=int, default=2) query_prune_parser = subparsers.add_parser("prune-old-query-projections") query_prune_parser.add_argument("--keep", type=int, default=2) query_prune_parser.add_argument("--execute", action="store_true") query_prune_parser.add_argument("--ack-delete-rebuildable-cache", action="store_true") args = parser.parse_args() if args.command == "status": result = status(args.include_counts) elif args.command == "backup": result = backup(args.output, args.verify) elif args.command == "checkpoint": result = checkpoint(args.mode, args.ack_maintenance_window) elif args.command == "cache-retention-report": result = cache_retention_report(args.limit) elif args.command == "prune-orphan-export-cache": result = prune_orphan_export_cache( dry_run=not args.execute, acknowledged=args.ack_delete_rebuildable_cache, ) elif args.command == "query-retention-report": result = query_projection_retention_report(args.keep) elif args.command == "prune-old-query-projections": result = prune_old_query_projections( dry_run=not args.execute, acknowledged=args.ack_delete_rebuildable_cache, keep=args.keep, ) else: result = prepare_runtime_layout(args.root) print(json.dumps(result, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()