Files
HM_project_Viewer_Board/scripts/sqlite_runtime_admin.py
T

569 lines
22 KiB
Python

from __future__ import annotations
import argparse
import json
import sqlite3
import sys
from datetime import datetime
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from runtime_config import ( # noqa: E402
BACKUP_DIR,
DB_PATH,
WAL_BLOCK_HEAVY_BYTES,
WAL_WARN_BYTES,
ensure_runtime_directories,
sqlite_runtime_status,
)
CACHE_TABLES = (
"wehago_compare_export_row_cache",
"wehago_compare_query_page_cache",
"wehago_compare_query_groups",
"wehago_compare_query_rows",
"wehago_metric_count_cache",
"wehago_pair_recommend_cache",
"wehago_raw_erp_trace_candidate_cache",
"wehago_result_row_cache",
"wehago_summary_range_cache",
)
SOURCE_TABLES = ("wehago_voucher_rows", "wehago_ledger_rows")
def _size(path: Path) -> int:
return path.stat().st_size if path.exists() else 0
def _table_counts(conn: sqlite3.Connection, tables: tuple[str, ...]) -> dict[str, int | None]:
existing = {
str(row[0])
for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
}
counts: dict[str, int | None] = {}
for table in tables:
if table not in existing:
counts[table] = None
continue
counts[table] = int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
return counts
def _table_storage(conn: sqlite3.Connection, tables: tuple[str, ...]) -> dict[str, dict[str, int | None]]:
existing = {
str(row[0])
for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
}
has_dbstat = True
try:
conn.execute("SELECT 1 FROM dbstat LIMIT 1").fetchone()
except sqlite3.DatabaseError:
has_dbstat = False
page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
storage: dict[str, dict[str, int | None]] = {}
for table in tables:
if table not in existing:
storage[table] = {"rows": None, "bytes": None}
continue
rows = int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
bytes_used: int | None = None
if has_dbstat:
try:
stat = conn.execute(
"SELECT SUM(pgsize) FROM dbstat WHERE name = ?",
(table,),
).fetchone()
bytes_used = int(stat[0] or 0)
except sqlite3.DatabaseError:
bytes_used = None
storage[table] = {"rows": rows, "bytes": bytes_used if has_dbstat else None}
if not has_dbstat:
for table in storage.values():
table["estimated_page_size"] = page_size
return storage
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
return bool(
conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
(table,),
).fetchone()
)
def cache_retention_report(limit: int) -> dict[str, Any]:
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
try:
if not _table_exists(conn, "wehago_compare_export_row_cache"):
return {"export_row_cache": {"exists": False}}
signatures = [
{
"fiscal_year": int(row[0]),
"snapshot_signature": str(row[1]),
"rows": int(row[2]),
}
for row in conn.execute(
"""
SELECT fiscal_year, snapshot_signature, COUNT(*) AS rows
FROM wehago_compare_export_row_cache
GROUP BY fiscal_year, snapshot_signature
ORDER BY fiscal_year DESC, rows DESC
LIMIT ?
""",
(limit,),
).fetchall()
]
orphan_rows = 0
if _table_exists(conn, "wehago_snapshot_status"):
orphan_rows = int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_export_row_cache c
LEFT JOIN wehago_snapshot_status s
ON s.fiscal_year = c.fiscal_year
AND s.snapshot_signature = c.snapshot_signature
AND s.state = 'ready'
WHERE s.fiscal_year IS NULL
"""
).fetchone()[0]
)
return {
"export_row_cache": {
"exists": True,
"signature_groups_sample": signatures,
"orphan_rows_not_matching_ready_snapshot": orphan_rows,
}
}
finally:
conn.close()
def prune_orphan_export_cache(dry_run: bool, acknowledged: bool) -> dict[str, Any]:
if not dry_run and not acknowledged:
raise RuntimeError(
"실제 삭제는 `--ack-delete-rebuildable-cache`를 함께 지정해야 합니다. "
"삭제 대상은 현재 ready 스냅샷 서명과 맞지 않는 재생성 가능 export-row 캐시입니다."
)
conn = sqlite3.connect(DB_PATH, timeout=30)
try:
if not _table_exists(conn, "wehago_compare_export_row_cache") or not _table_exists(
conn,
"wehago_snapshot_status",
):
return {"dry_run": dry_run, "candidate_rows": 0, "deleted_rows": 0}
candidate_rows = int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_export_row_cache c
LEFT JOIN wehago_snapshot_status s
ON s.fiscal_year = c.fiscal_year
AND s.snapshot_signature = c.snapshot_signature
AND s.state = 'ready'
WHERE s.fiscal_year IS NULL
"""
).fetchone()[0]
)
deleted_rows = 0
if not dry_run and candidate_rows:
conn.execute(
"""
DELETE FROM wehago_compare_export_row_cache
WHERE NOT EXISTS (
SELECT 1
FROM wehago_snapshot_status s
WHERE s.fiscal_year = wehago_compare_export_row_cache.fiscal_year
AND s.snapshot_signature = wehago_compare_export_row_cache.snapshot_signature
AND s.state = 'ready'
)
"""
)
deleted_rows = conn.total_changes
conn.commit()
return {"dry_run": dry_run, "candidate_rows": candidate_rows, "deleted_rows": deleted_rows}
finally:
conn.close()
def query_projection_retention_report(keep: int) -> dict[str, Any]:
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
conn.row_factory = sqlite3.Row
try:
if not _table_exists(conn, "wehago_compare_query_groups"):
return {"query_projection_cache": {"exists": False}}
groups = conn.execute(
"""
SELECT start_year, end_year, signature,
COUNT(*) AS group_rows,
MAX(updated_at) AS max_updated_at
FROM wehago_compare_query_groups
GROUP BY start_year, end_year, signature
ORDER BY start_year DESC, end_year DESC, max_updated_at DESC
"""
).fetchall()
keep = max(1, int(keep))
by_scope: dict[tuple[int, int], list[sqlite3.Row]] = {}
for row in groups:
by_scope.setdefault((int(row["start_year"]), int(row["end_year"])), []).append(row)
obsolete: list[sqlite3.Row] = []
retained: list[dict[str, Any]] = []
for scope, rows in by_scope.items():
for idx, row in enumerate(rows):
item = {
"start_year": int(row["start_year"]),
"end_year": int(row["end_year"]),
"signature": str(row["signature"] or ""),
"group_rows": int(row["group_rows"] or 0),
"max_updated_at": str(row["max_updated_at"] or ""),
"retained": idx < keep,
}
if idx < keep:
retained.append(item)
else:
obsolete.append(row)
obsolete_group_rows = sum(int(row["group_rows"] or 0) for row in obsolete)
obsolete_query_rows = 0
obsolete_page_rows = 0
for row in obsolete:
params = (
int(row["start_year"]),
int(row["end_year"]),
str(row["signature"] or ""),
)
if _table_exists(conn, "wehago_compare_query_rows"):
obsolete_query_rows += int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_query_rows
WHERE start_year = ? AND end_year = ? AND signature = ?
""",
params,
).fetchone()[0]
)
if _table_exists(conn, "wehago_compare_query_page_cache"):
obsolete_page_rows += int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_query_page_cache
WHERE start_year = ? AND end_year = ? AND signature = ?
""",
params,
).fetchone()[0]
)
return {
"query_projection_cache": {
"exists": True,
"keep_per_range": keep,
"range_signature_count": len(groups),
"obsolete_signature_count": len(obsolete),
"obsolete_group_rows": obsolete_group_rows,
"obsolete_query_rows": obsolete_query_rows,
"obsolete_page_rows": obsolete_page_rows,
"retained_sample": retained[:20],
}
}
finally:
conn.close()
def prune_old_query_projections(dry_run: bool, acknowledged: bool, keep: int) -> dict[str, Any]:
if not dry_run and not acknowledged:
raise RuntimeError(
"실제 삭제는 `--ack-delete-rebuildable-cache`를 함께 지정해야 합니다. "
"삭제 대상은 범위별 최신 N개를 제외한 재생성 가능 query projection 캐시입니다."
)
conn = sqlite3.connect(DB_PATH, timeout=30)
conn.row_factory = sqlite3.Row
try:
if not _table_exists(conn, "wehago_compare_query_groups"):
return {"dry_run": dry_run, "candidate_group_rows": 0, "candidate_query_rows": 0, "candidate_page_rows": 0}
keep = max(1, int(keep))
rows = conn.execute(
"""
SELECT start_year, end_year, signature, MAX(updated_at) AS max_updated_at
FROM wehago_compare_query_groups
GROUP BY start_year, end_year, signature
ORDER BY start_year, end_year, max_updated_at DESC
"""
).fetchall()
by_scope: dict[tuple[int, int], list[sqlite3.Row]] = {}
for row in rows:
by_scope.setdefault((int(row["start_year"]), int(row["end_year"])), []).append(row)
obsolete = [row for scope_rows in by_scope.values() for row in scope_rows[keep:]]
candidate_group_rows = 0
candidate_query_rows = 0
candidate_page_rows = 0
deleted_rows = 0
for row in obsolete:
params = (
int(row["start_year"]),
int(row["end_year"]),
str(row["signature"] or ""),
)
candidate_group_rows += int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_query_groups
WHERE start_year = ? AND end_year = ? AND signature = ?
""",
params,
).fetchone()[0]
)
if _table_exists(conn, "wehago_compare_query_rows"):
candidate_query_rows += int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_query_rows
WHERE start_year = ? AND end_year = ? AND signature = ?
""",
params,
).fetchone()[0]
)
if _table_exists(conn, "wehago_compare_query_page_cache"):
candidate_page_rows += int(
conn.execute(
"""
SELECT COUNT(*)
FROM wehago_compare_query_page_cache
WHERE start_year = ? AND end_year = ? AND signature = ?
""",
params,
).fetchone()[0]
)
if not dry_run:
for table in (
"wehago_compare_query_page_cache",
"wehago_compare_query_rows",
"wehago_compare_query_groups",
):
if not _table_exists(conn, table):
continue
before = conn.total_changes
conn.execute(
f"""
DELETE FROM {table}
WHERE start_year = ? AND end_year = ? AND signature = ?
""",
params,
)
deleted_rows += conn.total_changes - before
conn.commit()
return {
"dry_run": dry_run,
"keep_per_range": keep,
"obsolete_signature_count": len(obsolete),
"candidate_group_rows": candidate_group_rows,
"candidate_query_rows": candidate_query_rows,
"candidate_page_rows": candidate_page_rows,
"deleted_rows": deleted_rows,
}
finally:
conn.close()
def status(include_counts: bool) -> dict[str, Any]:
path = DB_PATH
payload: dict[str, Any] = {
"database_path": str(path),
"database_exists": path.exists(),
"files": {
"database_bytes": _size(path),
"wal_bytes": _size(path.with_name(path.name + "-wal")),
"shm_bytes": _size(path.with_name(path.name + "-shm")),
},
"runtime": sqlite_runtime_status(),
"thresholds": {
"wal_warn_bytes": WAL_WARN_BYTES,
"wal_block_heavy_bytes": WAL_BLOCK_HEAVY_BYTES,
},
}
if not path.exists():
return payload
conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5)
try:
payload["database"] = {
"journal_mode": str(conn.execute("PRAGMA journal_mode").fetchone()[0]),
"page_size": int(conn.execute("PRAGMA page_size").fetchone()[0]),
"page_count": int(conn.execute("PRAGMA page_count").fetchone()[0]),
"freelist_count": int(conn.execute("PRAGMA freelist_count").fetchone()[0]),
"wal_autocheckpoint": int(conn.execute("PRAGMA wal_autocheckpoint").fetchone()[0]),
}
if include_counts:
payload["source_table_counts"] = _table_counts(conn, SOURCE_TABLES)
payload["cache_table_counts"] = _table_counts(conn, CACHE_TABLES)
payload["cache_table_storage"] = _table_storage(conn, CACHE_TABLES)
finally:
conn.close()
wal_bytes = int(payload["files"]["wal_bytes"])
warnings: list[str] = []
if not payload["runtime"]["meets_minimum_safe_version"]:
warnings.append(
"SQLite 런타임이 3.51.3 미만입니다. WAL DB를 운영하기 전에 안전 런타임으로 교체하세요."
)
if wal_bytes >= WAL_BLOCK_HEAVY_BYTES:
warnings.append("WAL이 중단 기준을 넘었습니다. 신규 대량 캐시 재생성을 보류하고 유지보수 창을 확보하세요.")
elif wal_bytes >= WAL_WARN_BYTES:
warnings.append("WAL이 경고 기준을 넘었습니다. 긴 조회/쓰기 작업과 checkpoint 상태를 점검하세요.")
payload["warnings"] = warnings
return payload
def backup(output: Path | None, verify: str) -> dict[str, Any]:
if not DB_PATH.exists():
raise FileNotFoundError(DB_PATH)
ensure_runtime_directories()
output = output or BACKUP_DIR / f"data-migration-{datetime.now():%Y%m%d-%H%M%S}.sqlite3"
output.parent.mkdir(parents=True, exist_ok=True)
temp_path = output.with_name("." + output.name + ".tmp")
source_conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
target_conn = sqlite3.connect(temp_path)
last_reported_percent = -10
def report_progress(_status: int, remaining: int, total: int) -> None:
nonlocal last_reported_percent
percent = int((total - remaining) * 100 / total) if total else 100
reported_percent = min(100, (percent // 10) * 10)
if reported_percent > last_reported_percent:
print(f"backup copy progress: {reported_percent}%", file=sys.stderr, flush=True)
last_reported_percent = reported_percent
try:
source_conn.backup(target_conn, pages=8192, progress=report_progress)
check = "skipped"
if verify == "smoke":
target_conn.execute("PRAGMA schema_version").fetchone()
target_conn.execute("SELECT COUNT(*) FROM sqlite_master").fetchone()
check = "open-and-schema-readable"
elif verify != "none":
pragma = "integrity_check" if verify == "full" else "quick_check"
print(f"backup verification started: {pragma}", file=sys.stderr, flush=True)
check = str(target_conn.execute(f"PRAGMA {pragma}").fetchone()[0])
if check.lower() != "ok":
raise RuntimeError(f"백업 {pragma} 실패: {check}")
finally:
target_conn.close()
source_conn.close()
temp_path.replace(output)
return {"backup_path": str(output), "backup_bytes": _size(output), "verification": verify, "check_result": check}
def checkpoint(mode: str, acknowledged: bool) -> dict[str, Any]:
if not acknowledged:
raise RuntimeError(
"checkpoint는 운영 서버와 대량 작업을 중단한 유지보수 창에서만 실행하세요. "
"`--ack-maintenance-window`를 함께 지정해야 합니다."
)
conn = sqlite3.connect(DB_PATH, timeout=30)
try:
before = _size(DB_PATH.with_name(DB_PATH.name + "-wal"))
result = conn.execute(f"PRAGMA wal_checkpoint({mode.upper()})").fetchone()
after = _size(DB_PATH.with_name(DB_PATH.name + "-wal"))
finally:
conn.close()
return {"mode": mode, "checkpoint_result": list(result or ()), "wal_bytes_before": before, "wal_bytes_after": after}
def prepare_runtime_layout(root: Path) -> dict[str, str]:
paths = {
"db": root / "db",
"cache": root / "cache",
"backups": root / "backups",
"exports": root / "exports",
}
for path in paths.values():
path.mkdir(parents=True, exist_ok=True)
env_file = root / "runtime.env.example"
env_file.write_text(
"\n".join(
[
f"INTRANET_DB_PATH={paths['db'] / 'data.db'}",
f"INTRANET_BACKUP_DIR={paths['backups']}",
f"INTRANET_CACHE_ROOT={paths['cache']}",
f"INTRANET_COMPARE_EXPORT_DIR={paths['exports'] / 'wehago_compare'}",
"INTRANET_REQUIRE_SAFE_SQLITE=1",
f"WEHAGO_SOURCE_ROOT={Path.home() / 'WEHAGO_DB'}",
"",
]
),
encoding="utf-8",
)
return {key: str(path) for key, path in paths.items()} | {"env_example": str(env_file)}
def main() -> None:
parser = argparse.ArgumentParser(description="SQLite/WAL runtime inspection and maintenance helpers.")
subparsers = parser.add_subparsers(dest="command", required=True)
status_parser = subparsers.add_parser("status")
status_parser.add_argument("--include-counts", action="store_true")
backup_parser = subparsers.add_parser("backup")
backup_parser.add_argument("--output", type=Path)
backup_parser.add_argument(
"--verify",
choices=("smoke", "quick", "full", "none"),
default="smoke",
help="smoke is suitable for container trials; use quick/full during a maintenance window.",
)
checkpoint_parser = subparsers.add_parser("checkpoint")
checkpoint_parser.add_argument("--mode", choices=("passive", "full", "restart", "truncate"), default="passive")
checkpoint_parser.add_argument("--ack-maintenance-window", action="store_true")
layout_parser = subparsers.add_parser("prepare-layout")
layout_parser.add_argument("--root", type=Path, default=Path.home() / "intranet-runtime")
report_parser = subparsers.add_parser("cache-retention-report")
report_parser.add_argument("--limit", type=int, default=20)
prune_parser = subparsers.add_parser("prune-orphan-export-cache")
prune_parser.add_argument("--execute", action="store_true")
prune_parser.add_argument("--ack-delete-rebuildable-cache", action="store_true")
query_report_parser = subparsers.add_parser("query-retention-report")
query_report_parser.add_argument("--keep", type=int, default=2)
query_prune_parser = subparsers.add_parser("prune-old-query-projections")
query_prune_parser.add_argument("--keep", type=int, default=2)
query_prune_parser.add_argument("--execute", action="store_true")
query_prune_parser.add_argument("--ack-delete-rebuildable-cache", action="store_true")
args = parser.parse_args()
if args.command == "status":
result = status(args.include_counts)
elif args.command == "backup":
result = backup(args.output, args.verify)
elif args.command == "checkpoint":
result = checkpoint(args.mode, args.ack_maintenance_window)
elif args.command == "cache-retention-report":
result = cache_retention_report(args.limit)
elif args.command == "prune-orphan-export-cache":
result = prune_orphan_export_cache(
dry_run=not args.execute,
acknowledged=args.ack_delete_rebuildable_cache,
)
elif args.command == "query-retention-report":
result = query_projection_retention_report(args.keep)
elif args.command == "prune-old-query-projections":
result = prune_old_query_projections(
dry_run=not args.execute,
acknowledged=args.ack_delete_rebuildable_cache,
keep=args.keep,
)
else:
result = prepare_runtime_layout(args.root)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()