Update intranet tools and voucher comparison
This commit is contained in:
b17301
@@ -0,0 +1,427 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font, PatternFill
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from runtime_config import DB_PATH # noqa: E402
|
||||
|
||||
EXPORT_DIR = ROOT / "static" / "exports" / "wehago_compare"
|
||||
YEAR = 2025
|
||||
WEHAGO_STATUSES = ("voucher_matched", "voucher_unmatched", "voucher_recheck")
|
||||
|
||||
|
||||
def clean(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def key_from_display(year: int, ledger_date: str, voucher_no: str) -> str:
|
||||
voucher = clean(voucher_no)
|
||||
if re.fullmatch(r"\d{8}-\d{5}", voucher):
|
||||
return voucher
|
||||
match = re.fullmatch(r"(\d{4})-(\d{2})-(\d{5})", voucher)
|
||||
if match:
|
||||
return f"{match.group(1)}{match.group(2)}-{match.group(3)}"
|
||||
date_text = clean(ledger_date)
|
||||
date_match = re.search(r"(\d{1,2})[-./](\d{1,2})", date_text)
|
||||
voucher_digits = re.sub(r"\D+", "", voucher)
|
||||
if date_match and voucher_digits:
|
||||
month = int(date_match.group(1))
|
||||
day = int(date_match.group(2))
|
||||
return f"{int(year):04d}{month:02d}{day:02d}-{int(voucher_digits):05d}"
|
||||
return ""
|
||||
|
||||
|
||||
def latest_projection_signature(cur: sqlite3.Cursor) -> str:
|
||||
row = cur.execute(
|
||||
"""
|
||||
SELECT payload_json
|
||||
FROM wehago_action_history
|
||||
WHERE action_type = 'auto_recheck_promote'
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
if row:
|
||||
try:
|
||||
payload = json.loads(row[0] or "{}")
|
||||
signature = clean(payload.get("signature"))
|
||||
if signature:
|
||||
exists = cur.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(YEAR, YEAR, signature),
|
||||
).fetchone()
|
||||
if exists:
|
||||
return signature
|
||||
except Exception:
|
||||
pass
|
||||
row = cur.execute(
|
||||
"""
|
||||
SELECT signature, MAX(updated_at) AS max_updated_at
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ? AND end_year = ? AND signature LIKE 'compare-query-v5|%'
|
||||
GROUP BY signature
|
||||
ORDER BY max_updated_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(YEAR, YEAR),
|
||||
).fetchone()
|
||||
return clean(row[0]) if row else ""
|
||||
|
||||
|
||||
def append_rows(ws, rows: list[list[Any]]) -> None:
|
||||
for row in rows:
|
||||
ws.append(row)
|
||||
|
||||
|
||||
def style_sheet(ws) -> None:
|
||||
header_fill = PatternFill("solid", fgColor="D9EAF7")
|
||||
for cell in ws[1]:
|
||||
cell.font = Font(bold=True)
|
||||
cell.fill = header_fill
|
||||
ws.freeze_panes = "A2"
|
||||
ws.auto_filter.ref = ws.dimensions
|
||||
for col_idx, column_cells in enumerate(ws.columns, start=1):
|
||||
max_len = 10
|
||||
for cell in column_cells:
|
||||
max_len = max(max_len, min(len(clean(cell.value)), 80))
|
||||
ws.column_dimensions[get_column_letter(col_idx)].width = max_len + 2
|
||||
|
||||
|
||||
def main() -> None:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
signature = latest_projection_signature(cur)
|
||||
if not signature:
|
||||
raise RuntimeError("compare query projection signature not found")
|
||||
|
||||
db_rows = cur.execute(
|
||||
"""
|
||||
SELECT fiscal_year, voucher_no, status, voucher_row_count, ledger_row_count,
|
||||
voucher_debit, voucher_credit, ledger_debit, ledger_credit,
|
||||
voucher_accounts, ledger_accounts, voucher_vendors, ledger_vendors, notes
|
||||
FROM wehago_comparison_results
|
||||
WHERE fiscal_year = ?
|
||||
AND status <> 'voucher_only'
|
||||
ORDER BY voucher_no
|
||||
""",
|
||||
(YEAR,),
|
||||
).fetchall()
|
||||
db_by_key = {clean(row["voucher_no"]): dict(row) for row in db_rows if clean(row["voucher_no"])}
|
||||
comparison_total = cur.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS count_all,
|
||||
SUM(CASE WHEN status = 'voucher_only' THEN 1 ELSE 0 END) AS voucher_only_count
|
||||
FROM wehago_comparison_results
|
||||
WHERE fiscal_year = ?
|
||||
""",
|
||||
(YEAR,),
|
||||
).fetchone()
|
||||
|
||||
ledger_source = cur.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS row_count,
|
||||
COUNT(DISTINCT fiscal_year || '|' || compare_voucher_no) AS voucher_count
|
||||
FROM wehago_ledger_rows
|
||||
WHERE fiscal_year = ? AND COALESCE(compare_voucher_no, '') <> ''
|
||||
""",
|
||||
(YEAR,),
|
||||
).fetchone()
|
||||
voucher_source = cur.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS row_count,
|
||||
COUNT(DISTINCT fiscal_year || '|' || compare_voucher_no) AS voucher_count
|
||||
FROM wehago_voucher_rows
|
||||
WHERE fiscal_year = ? AND COALESCE(compare_voucher_no, '') <> ''
|
||||
""",
|
||||
(YEAR,),
|
||||
).fetchone()
|
||||
|
||||
ui_rows = cur.execute(
|
||||
f"""
|
||||
SELECT status_key, group_index, fiscal_year, ledger_date, proof_date, voucher_no, draft_no,
|
||||
ledger_row_count, voucher_row_count, ledger_debit, ledger_credit, voucher_debit, voucher_credit,
|
||||
ledger_accounts, voucher_accounts, ledger_vendors, voucher_vendors, review_reason
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
AND status_key IN ({','.join('?' for _ in WEHAGO_STATUSES)})
|
||||
ORDER BY status_key, group_index
|
||||
""",
|
||||
(YEAR, YEAR, signature, *WEHAGO_STATUSES),
|
||||
).fetchall()
|
||||
|
||||
ui_group_records: list[dict[str, Any]] = []
|
||||
for row in ui_rows:
|
||||
record = dict(row)
|
||||
record["compare_key"] = key_from_display(record["fiscal_year"], record["ledger_date"], record["voucher_no"])
|
||||
ui_group_records.append(record)
|
||||
|
||||
section_counts = Counter(record["status_key"] for record in ui_group_records)
|
||||
ui_by_key: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
bad_key_records: list[dict[str, Any]] = []
|
||||
for record in ui_group_records:
|
||||
key = clean(record.get("compare_key"))
|
||||
if key:
|
||||
ui_by_key[key].append(record)
|
||||
else:
|
||||
bad_key_records.append(record)
|
||||
|
||||
db_keys = set(db_by_key)
|
||||
ui_keys = set(ui_by_key)
|
||||
duplicate_ui = {key: records for key, records in ui_by_key.items() if len(records) > 1}
|
||||
ui_not_in_db = {key: records for key, records in ui_by_key.items() if key not in db_keys}
|
||||
db_not_in_ui = {key: db_by_key[key] for key in sorted(db_keys - ui_keys)}
|
||||
|
||||
section_sum = sum(section_counts.get(status, 0) for status in WEHAGO_STATUSES)
|
||||
db_count = len(db_by_key)
|
||||
ui_unique_count = len(ui_keys)
|
||||
duplicate_extra_count = section_sum - ui_unique_count
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "요약"
|
||||
append_rows(
|
||||
ws,
|
||||
[
|
||||
["항목", "값", "설명"],
|
||||
["분석 기준 연도", YEAR, ""],
|
||||
["사용 projection signature", signature, "화면 WEHAGO/Recheck 섹션에 사용된 최신 projection"],
|
||||
["DB WEHAGO 전표 수", db_count, "wehago_comparison_results의 fiscal_year+voucher_no 기준"],
|
||||
["DB 비교결과 전체 전표 수", int(comparison_total["count_all"] or 0), "ERP-only(voucher_only)를 포함한 전체 비교 결과"],
|
||||
["DB 비교결과 중 ERP-only 수", int(comparison_total["voucher_only_count"] or 0), "WEHAGO 전표 수 비교에서는 제외"],
|
||||
["DB WEHAGO 원장 전표 수", int(ledger_source["voucher_count"] or 0), "wehago_ledger_rows의 fiscal_year+compare_voucher_no distinct"],
|
||||
["DB WEHAGO 원장 행 수", int(ledger_source["row_count"] or 0), "wehago_ledger_rows 행 수"],
|
||||
["보조: wehago_voucher_rows 전표 수", int(voucher_source["voucher_count"] or 0), "ERP/증빙성 원천 행으로 보이는 테이블의 distinct 전표 수"],
|
||||
["WEHAGO", section_counts.get("voucher_matched", 0), "화면 WEHAGO 그룹 수"],
|
||||
["WEHAGO Unmatched", section_counts.get("voucher_unmatched", 0), "화면 WEHAGO Unmatched 그룹 수"],
|
||||
["WEHAGO Recheck", section_counts.get("voucher_recheck", 0), "화면 WEHAGO Recheck 그룹 수"],
|
||||
["화면 3개 섹션 합계", section_sum, "WEHAGO + WEHAGO Unmatched + WEHAGO Recheck"],
|
||||
["차이(화면 합계 - DB WEHAGO)", section_sum - db_count, "양수면 화면 그룹 집계가 DB 전표 수보다 많음"],
|
||||
["화면 고유 WEHAGO 전표 수", ui_unique_count, "ledger_date+voucher_no를 YYYYMMDD-전표번호로 환산한 distinct"],
|
||||
["차이(화면 고유 전표 - DB WEHAGO)", ui_unique_count - db_count, "전표번호 기준 순수 누락/추가 차이"],
|
||||
["중복 그룹 초과분", duplicate_extra_count, "같은 WEHAGO 전표가 여러 그룹으로 나뉘어 합계에 중복 반영된 수"],
|
||||
["키 생성 불가 그룹", len(bad_key_records), "ledger_date/voucher_no 조합으로 DB 전표번호를 만들 수 없는 그룹"],
|
||||
],
|
||||
)
|
||||
style_sheet(ws)
|
||||
|
||||
ws = wb.create_sheet("차이 사유")
|
||||
reason_rows = [
|
||||
["사유", "건수", "해석"],
|
||||
[
|
||||
"같은 WEHAGO 전표가 화면에서 복수 그룹으로 집계됨",
|
||||
duplicate_extra_count,
|
||||
"전표 하나가 복수 ERP 가전표/후보/라인 그룹으로 분리되면 화면 섹션 합계는 DB 전표 수보다 커집니다.",
|
||||
],
|
||||
[
|
||||
"화면에는 있으나 DB comparison_results 전표번호와 직접 대응되지 않음",
|
||||
len(ui_not_in_db),
|
||||
"표시 전표번호를 YYYYMMDD-전표번호로 환산해도 DB 전표번호 집합에 없는 경우입니다.",
|
||||
],
|
||||
[
|
||||
"DB에는 있으나 화면 3개 WEHAGO 섹션에 없음",
|
||||
len(db_not_in_ui),
|
||||
"DB 비교 결과에는 있으나 현재 projection의 WEHAGO/Unmatched/Recheck 그룹에는 없는 경우입니다.",
|
||||
],
|
||||
[
|
||||
"전표번호 키 생성 불가",
|
||||
len(bad_key_records),
|
||||
"화면 그룹의 일자 또는 전표번호가 비어 있거나 형식이 달라 비교 키를 만들지 못한 경우입니다.",
|
||||
],
|
||||
]
|
||||
append_rows(ws, reason_rows)
|
||||
style_sheet(ws)
|
||||
|
||||
ws = wb.create_sheet("중복 그룹 상세")
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
"compare_key",
|
||||
"그룹 수",
|
||||
"status 목록",
|
||||
"전표 표시값",
|
||||
"일자 목록",
|
||||
"가전표번호/ERP 전표",
|
||||
"WEHAGO 계정",
|
||||
"ERP 계정",
|
||||
"검토 사유",
|
||||
]],
|
||||
)
|
||||
for key, records in sorted(duplicate_ui.items(), key=lambda item: (-len(item[1]), item[0])):
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
key,
|
||||
len(records),
|
||||
", ".join(sorted({clean(r.get("status_key")) for r in records})),
|
||||
", ".join(sorted({clean(r.get("voucher_no")) for r in records if clean(r.get("voucher_no"))}))[:300],
|
||||
", ".join(sorted({clean(r.get("ledger_date")) for r in records if clean(r.get("ledger_date"))}))[:300],
|
||||
", ".join(clean(r.get("draft_no")) for r in records if clean(r.get("draft_no")))[:1000],
|
||||
" | ".join(clean(r.get("ledger_accounts")) for r in records if clean(r.get("ledger_accounts")))[:1000],
|
||||
" | ".join(clean(r.get("voucher_accounts")) for r in records if clean(r.get("voucher_accounts")))[:1000],
|
||||
" | ".join(clean(r.get("review_reason")) for r in records if clean(r.get("review_reason")))[:1000],
|
||||
]],
|
||||
)
|
||||
style_sheet(ws)
|
||||
|
||||
ws = wb.create_sheet("화면만 있음")
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
"compare_key",
|
||||
"그룹 수",
|
||||
"status 목록",
|
||||
"일자 목록",
|
||||
"전표 표시값",
|
||||
"가전표번호/ERP 전표",
|
||||
"WEHAGO 계정",
|
||||
"ERP 계정",
|
||||
"검토 사유",
|
||||
]],
|
||||
)
|
||||
for key, records in sorted(ui_not_in_db.items()):
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
key,
|
||||
len(records),
|
||||
", ".join(sorted({clean(r.get("status_key")) for r in records})),
|
||||
", ".join(sorted({clean(r.get("ledger_date")) for r in records if clean(r.get("ledger_date"))}))[:300],
|
||||
", ".join(sorted({clean(r.get("voucher_no")) for r in records if clean(r.get("voucher_no"))}))[:300],
|
||||
", ".join(clean(r.get("draft_no")) for r in records if clean(r.get("draft_no")))[:1000],
|
||||
" | ".join(clean(r.get("ledger_accounts")) for r in records if clean(r.get("ledger_accounts")))[:1000],
|
||||
" | ".join(clean(r.get("voucher_accounts")) for r in records if clean(r.get("voucher_accounts")))[:1000],
|
||||
" | ".join(clean(r.get("review_reason")) for r in records if clean(r.get("review_reason")))[:1000],
|
||||
]],
|
||||
)
|
||||
style_sheet(ws)
|
||||
|
||||
ws = wb.create_sheet("DB만 있음")
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
"compare_key",
|
||||
"DB status",
|
||||
"voucher_row_count",
|
||||
"ledger_row_count",
|
||||
"WEHAGO 차변",
|
||||
"WEHAGO 대변",
|
||||
"ERP 차변",
|
||||
"ERP 대변",
|
||||
"WEHAGO 계정",
|
||||
"ERP 계정",
|
||||
"비고",
|
||||
]],
|
||||
)
|
||||
for key, row in db_not_in_ui.items():
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
key,
|
||||
clean(row.get("status")),
|
||||
row.get("ledger_row_count"),
|
||||
row.get("voucher_row_count"),
|
||||
row.get("ledger_debit"),
|
||||
row.get("ledger_credit"),
|
||||
row.get("voucher_debit"),
|
||||
row.get("voucher_credit"),
|
||||
clean(row.get("ledger_accounts")),
|
||||
clean(row.get("voucher_accounts")),
|
||||
clean(row.get("notes")),
|
||||
]],
|
||||
)
|
||||
style_sheet(ws)
|
||||
|
||||
ws = wb.create_sheet("섹션 원자료")
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
"status",
|
||||
"compare_key",
|
||||
"fiscal_year",
|
||||
"ledger_date",
|
||||
"voucher_no",
|
||||
"draft_no",
|
||||
"ledger_row_count",
|
||||
"voucher_row_count",
|
||||
"WEHAGO 차변",
|
||||
"WEHAGO 대변",
|
||||
"ERP 차변",
|
||||
"ERP 대변",
|
||||
"WEHAGO 계정",
|
||||
"ERP 계정",
|
||||
"WEHAGO 거래처",
|
||||
"ERP 거래처",
|
||||
"review_reason",
|
||||
]],
|
||||
)
|
||||
for record in ui_group_records:
|
||||
append_rows(
|
||||
ws,
|
||||
[[
|
||||
record.get("status_key"),
|
||||
record.get("compare_key"),
|
||||
record.get("fiscal_year"),
|
||||
record.get("ledger_date"),
|
||||
record.get("voucher_no"),
|
||||
record.get("draft_no"),
|
||||
record.get("ledger_row_count"),
|
||||
record.get("voucher_row_count"),
|
||||
record.get("ledger_debit"),
|
||||
record.get("ledger_credit"),
|
||||
record.get("voucher_debit"),
|
||||
record.get("voucher_credit"),
|
||||
record.get("ledger_accounts"),
|
||||
record.get("voucher_accounts"),
|
||||
record.get("ledger_vendors"),
|
||||
record.get("voucher_vendors"),
|
||||
record.get("review_reason"),
|
||||
]],
|
||||
)
|
||||
style_sheet(ws)
|
||||
|
||||
EXPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
file_name = f"wehago_voucher_coverage_{YEAR}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx"
|
||||
output_path = EXPORT_DIR / file_name
|
||||
wb.save(output_path)
|
||||
conn.close()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"file": str(output_path),
|
||||
"download_url": f"/static/exports/wehago_compare/{file_name}",
|
||||
"db_count": db_count,
|
||||
"section_sum": section_sum,
|
||||
"diff": section_sum - db_count,
|
||||
"ui_unique_count": ui_unique_count,
|
||||
"unique_diff": ui_unique_count - db_count,
|
||||
"duplicate_extra_count": duplicate_extra_count,
|
||||
"ui_not_in_db": len(ui_not_in_db),
|
||||
"db_not_in_ui": len(db_not_in_ui),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,506 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from runtime_config import DB_PATH # noqa: E402
|
||||
from scripts.project_export_cache_ranges import ( # noqa: E402
|
||||
current_ready_export_signature,
|
||||
latest_export_signature,
|
||||
project_range,
|
||||
projection_signature,
|
||||
)
|
||||
from wehago_compare import ( # noqa: E402
|
||||
QUERY_PROJECTION_VERSION,
|
||||
_parse_row_date_with_year,
|
||||
_raw_erp_entry_date_values,
|
||||
_raw_erp_trace_prefilter,
|
||||
_raw_erp_trace_prefilter_score,
|
||||
_raw_erp_trace_score,
|
||||
clean,
|
||||
parse_amount,
|
||||
)
|
||||
|
||||
|
||||
TRACE_LOGIC_VERSION = "raw-erp-trace-candidate-v1"
|
||||
SOURCE_STATUSES = ("voucher_unmatched", "voucher_recheck")
|
||||
|
||||
|
||||
def _connect(db_path: Path = DB_PATH) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(db_path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA busy_timeout = 30000")
|
||||
return conn
|
||||
|
||||
|
||||
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS wehago_raw_erp_trace_candidate_cache (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
candidate_key TEXT NOT NULL DEFAULT '',
|
||||
fiscal_year INTEGER NOT NULL,
|
||||
source_mode TEXT NOT NULL DEFAULT '',
|
||||
source_signature TEXT NOT NULL DEFAULT '',
|
||||
logic_version TEXT NOT NULL DEFAULT '',
|
||||
status_key TEXT NOT NULL DEFAULT '',
|
||||
group_index INTEGER NOT NULL DEFAULT 0,
|
||||
row_index INTEGER NOT NULL DEFAULT 0,
|
||||
ledger_date TEXT NOT NULL DEFAULT '',
|
||||
voucher_no TEXT NOT NULL DEFAULT '',
|
||||
ledger_account_name TEXT NOT NULL DEFAULT '',
|
||||
ledger_vendor TEXT NOT NULL DEFAULT '',
|
||||
ledger_desc TEXT NOT NULL DEFAULT '',
|
||||
ledger_amount REAL NOT NULL DEFAULT 0,
|
||||
erp_draft_no TEXT NOT NULL DEFAULT '',
|
||||
erp_confirmed_no TEXT NOT NULL DEFAULT '',
|
||||
erp_account_name TEXT NOT NULL DEFAULT '',
|
||||
erp_vendor TEXT NOT NULL DEFAULT '',
|
||||
erp_desc TEXT NOT NULL DEFAULT '',
|
||||
erp_amount REAL NOT NULL DEFAULT 0,
|
||||
amount_field TEXT NOT NULL DEFAULT '',
|
||||
score REAL NOT NULL DEFAULT 0,
|
||||
matched_case TEXT NOT NULL DEFAULT '',
|
||||
review_reason TEXT NOT NULL DEFAULT '',
|
||||
candidate_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""
|
||||
)
|
||||
columns = {
|
||||
str(row["name"] or "")
|
||||
for row in conn.execute("PRAGMA table_info(wehago_raw_erp_trace_candidate_cache)").fetchall()
|
||||
}
|
||||
if "candidate_key" not in columns:
|
||||
conn.execute(
|
||||
"ALTER TABLE wehago_raw_erp_trace_candidate_cache ADD COLUMN candidate_key TEXT NOT NULL DEFAULT ''"
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_source
|
||||
ON wehago_raw_erp_trace_candidate_cache(
|
||||
fiscal_year, source_mode, source_signature, logic_version, status_key, group_index
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_voucher
|
||||
ON wehago_raw_erp_trace_candidate_cache(fiscal_year, ledger_date, voucher_no)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_score
|
||||
ON wehago_raw_erp_trace_candidate_cache(fiscal_year, score DESC)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_wehago_raw_trace_candidate_key
|
||||
ON wehago_raw_erp_trace_candidate_cache(candidate_key)
|
||||
WHERE candidate_key <> ''
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _source_signature(conn: sqlite3.Connection, year: int, source_mode: str) -> str:
|
||||
if source_mode == "current":
|
||||
yearly_signature = current_ready_export_signature(conn, year)
|
||||
return projection_signature({year: yearly_signature}, year, year, allow_stale=False)
|
||||
if source_mode == "stale-diagnostic":
|
||||
yearly_signature = latest_export_signature(conn, year, allow_stale=True)
|
||||
return projection_signature({year: yearly_signature}, year, year, allow_stale=True)
|
||||
raise ValueError(f"Unknown source mode: {source_mode}")
|
||||
|
||||
|
||||
def _ensure_projection(conn: sqlite3.Connection, year: int, source_mode: str) -> str:
|
||||
signature = _source_signature(conn, year, source_mode)
|
||||
exists = conn.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ?
|
||||
AND end_year = ?
|
||||
AND signature = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(year, year, signature),
|
||||
).fetchone()
|
||||
if exists is not None:
|
||||
return signature
|
||||
project_range(conn, year, year, allow_stale=(source_mode == "stale-diagnostic"))
|
||||
return signature
|
||||
|
||||
|
||||
def _entry_amount_index(conn: sqlite3.Connection, year: int) -> dict[float, list[dict[str, Any]]]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT fiscal_year, proof_date, confirmed_no, draft_no, account_code, account_name,
|
||||
debit_supply, debit_tax, credit_supply, credit_tax,
|
||||
support_dept_name, cost_dept_name, desc1, desc2, vendor_name, management_item
|
||||
FROM wehago_voucher_rows
|
||||
WHERE fiscal_year = ?
|
||||
""",
|
||||
(year,),
|
||||
)
|
||||
indexed: dict[float, list[dict[str, Any]]] = defaultdict(list)
|
||||
fields = (
|
||||
("debit_supply", "debit", False),
|
||||
("credit_supply", "credit", False),
|
||||
("debit_tax", "debit", True),
|
||||
("credit_tax", "credit", True),
|
||||
)
|
||||
for row in rows:
|
||||
raw = dict(row)
|
||||
for field, side, tax_evidence in fields:
|
||||
amount = parse_amount(raw.get(field))
|
||||
if abs(amount) < 0.5:
|
||||
continue
|
||||
entry = dict(raw)
|
||||
entry["_raw_amount_field"] = field
|
||||
entry["_raw_amount_side"] = side
|
||||
entry["_raw_amount_value"] = amount
|
||||
entry["_raw_tax_evidence"] = tax_evidence
|
||||
entry["_raw_entry_dates"] = tuple(_raw_erp_entry_date_values(entry))
|
||||
indexed[round(abs(amount), 2)].append(entry)
|
||||
return dict(indexed)
|
||||
|
||||
|
||||
def _group_filters(args: argparse.Namespace) -> tuple[str, list[Any]]:
|
||||
filters: list[str] = []
|
||||
params: list[Any] = []
|
||||
if args.voucher_no:
|
||||
placeholders = ",".join("?" for _ in args.voucher_no)
|
||||
filters.append(f"g.voucher_no IN ({placeholders})")
|
||||
params.extend(args.voucher_no)
|
||||
if args.ledger_date:
|
||||
placeholders = ",".join("?" for _ in args.ledger_date)
|
||||
filters.append(f"g.ledger_date IN ({placeholders})")
|
||||
params.extend(args.ledger_date)
|
||||
if not filters:
|
||||
return "", []
|
||||
return " AND " + " AND ".join(filters), params
|
||||
|
||||
|
||||
def _load_source_groups(
|
||||
conn: sqlite3.Connection,
|
||||
year: int,
|
||||
signature: str,
|
||||
args: argparse.Namespace,
|
||||
) -> list[dict[str, Any]]:
|
||||
filter_sql, filter_params = _group_filters(args)
|
||||
limit_sql = " LIMIT ?" if args.limit_groups else ""
|
||||
offset_sql = " OFFSET ?" if args.limit_groups and args.group_offset else ""
|
||||
params: list[Any] = [year, year, signature, *SOURCE_STATUSES, *filter_params]
|
||||
if args.limit_groups:
|
||||
params.append(int(args.limit_groups))
|
||||
if args.group_offset:
|
||||
params.append(int(args.group_offset))
|
||||
groups = conn.execute(
|
||||
f"""
|
||||
SELECT g.status_key, g.group_index, g.fiscal_year, g.ledger_date, g.voucher_no, g.draft_no
|
||||
FROM wehago_compare_query_groups AS g
|
||||
WHERE g.start_year = ?
|
||||
AND g.end_year = ?
|
||||
AND g.signature = ?
|
||||
AND g.status_key IN ({','.join('?' for _ in SOURCE_STATUSES)})
|
||||
{filter_sql}
|
||||
ORDER BY g.status_key ASC, g.group_index ASC
|
||||
{limit_sql}
|
||||
{offset_sql}
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
if not groups:
|
||||
return []
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
for group_row in groups:
|
||||
row_items = conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM wehago_compare_query_rows
|
||||
WHERE start_year = ?
|
||||
AND end_year = ?
|
||||
AND signature = ?
|
||||
AND status_key = ?
|
||||
AND group_index = ?
|
||||
ORDER BY row_index ASC
|
||||
""",
|
||||
(year, year, signature, group_row["status_key"], group_row["group_index"]),
|
||||
).fetchall()
|
||||
rows = []
|
||||
for row in row_items:
|
||||
data = dict(row)
|
||||
data.setdefault("group_voucher_no", group_row["voucher_no"])
|
||||
data.setdefault("group_draft_no", group_row["draft_no"])
|
||||
if not clean(data.get("voucher_no")):
|
||||
data["voucher_no"] = group_row["voucher_no"]
|
||||
if not clean(data.get("draft_no")):
|
||||
data["draft_no"] = group_row["draft_no"]
|
||||
rows.append(data)
|
||||
result.append({"group": dict(group_row), "rows": rows})
|
||||
return result
|
||||
|
||||
|
||||
def _ledger_amount(row: dict[str, Any]) -> float:
|
||||
return round(max(abs(parse_amount(row.get("ledger_debit"))), abs(parse_amount(row.get("ledger_credit")))), 2)
|
||||
|
||||
|
||||
def _entries_in_date_window(
|
||||
ledger_row: dict[str, Any],
|
||||
entries: list[dict[str, Any]],
|
||||
window_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
if window_days < 0:
|
||||
return entries
|
||||
ledger_dates = [
|
||||
value
|
||||
for value in (
|
||||
_parse_row_date_with_year(ledger_row, "ledger_date"),
|
||||
_parse_row_date_with_year(ledger_row, "proof_date"),
|
||||
)
|
||||
if value
|
||||
]
|
||||
if not ledger_dates:
|
||||
return entries
|
||||
filtered: list[dict[str, Any]] = []
|
||||
for entry in entries:
|
||||
entry_dates = entry.get("_raw_entry_dates") or ()
|
||||
if any(abs((ledger_date - entry_date).days) <= window_days for ledger_date in ledger_dates for entry_date in entry_dates):
|
||||
filtered.append(entry)
|
||||
return filtered
|
||||
|
||||
|
||||
def _matched_case(score: float, candidate: dict[str, Any]) -> str:
|
||||
existing = clean(candidate.get("matched_case"))
|
||||
if existing:
|
||||
return existing
|
||||
if score >= 105:
|
||||
return "RAW_ERP_HIGH_CONFIDENCE_TRACE_CANDIDATE"
|
||||
return "RAW_ERP_SOURCE_TRACE_CANDIDATE"
|
||||
|
||||
|
||||
def _candidate_identity(
|
||||
year: int,
|
||||
source_mode: str,
|
||||
source_signature: str,
|
||||
logic_version: str,
|
||||
status_key: str,
|
||||
group_index: int,
|
||||
row_index: int,
|
||||
candidate: dict[str, Any],
|
||||
) -> str:
|
||||
raw = "|".join(
|
||||
clean(part)
|
||||
for part in (
|
||||
year,
|
||||
source_mode,
|
||||
source_signature,
|
||||
logic_version,
|
||||
status_key,
|
||||
group_index,
|
||||
row_index,
|
||||
candidate.get("draft_no"),
|
||||
candidate.get("voucher_confirmed_no"),
|
||||
candidate.get("voucher_account_name"),
|
||||
candidate.get("voucher_debit"),
|
||||
candidate.get("voucher_credit"),
|
||||
)
|
||||
)
|
||||
return hashlib.sha1(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _insert_candidates(
|
||||
conn: sqlite3.Connection,
|
||||
year: int,
|
||||
source_mode: str,
|
||||
source_signature: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT OR REPLACE INTO wehago_raw_erp_trace_candidate_cache (
|
||||
candidate_key, fiscal_year, source_mode, source_signature, logic_version, status_key,
|
||||
group_index, row_index, ledger_date, voucher_no,
|
||||
ledger_account_name, ledger_vendor, ledger_desc, ledger_amount,
|
||||
erp_draft_no, erp_confirmed_no, erp_account_name, erp_vendor, erp_desc,
|
||||
erp_amount, amount_field, score, matched_case, review_reason, candidate_json
|
||||
)
|
||||
VALUES (
|
||||
:candidate_key, :fiscal_year, :source_mode, :source_signature, :logic_version, :status_key,
|
||||
:group_index, :row_index, :ledger_date, :voucher_no,
|
||||
:ledger_account_name, :ledger_vendor, :ledger_desc, :ledger_amount,
|
||||
:erp_draft_no, :erp_confirmed_no, :erp_account_name, :erp_vendor, :erp_desc,
|
||||
:erp_amount, :amount_field, :score, :matched_case, :review_reason, :candidate_json
|
||||
)
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def build_candidates(args: argparse.Namespace) -> dict[str, Any]:
|
||||
conn = _connect(args.db)
|
||||
try:
|
||||
_ensure_schema(conn)
|
||||
source_signature = _ensure_projection(conn, args.year, args.source)
|
||||
if args.reset:
|
||||
conn.execute(
|
||||
"""
|
||||
DELETE FROM wehago_raw_erp_trace_candidate_cache
|
||||
WHERE fiscal_year = ?
|
||||
AND source_mode = ?
|
||||
AND source_signature = ?
|
||||
AND logic_version = ?
|
||||
""",
|
||||
(args.year, args.source, source_signature, TRACE_LOGIC_VERSION),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
amount_index = _entry_amount_index(conn, args.year)
|
||||
source_groups = _load_source_groups(conn, args.year, source_signature, args)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
scored_rows = 0
|
||||
skipped_common_amounts = 0
|
||||
|
||||
for source_group in source_groups:
|
||||
group_meta = source_group["group"]
|
||||
status_key = clean(group_meta.get("status_key"))
|
||||
group_index = int(group_meta.get("group_index") or 0)
|
||||
for ledger_row in source_group["rows"]:
|
||||
amount = _ledger_amount(ledger_row)
|
||||
if amount <= 0:
|
||||
continue
|
||||
entries = list(amount_index.get(amount, []) or [])
|
||||
if not entries:
|
||||
continue
|
||||
if len(entries) > args.date_prefilter_threshold:
|
||||
date_entries = _entries_in_date_window(ledger_row, entries, args.date_window_days)
|
||||
if date_entries:
|
||||
entries = date_entries
|
||||
if len(entries) > args.prefilter_threshold:
|
||||
entries = [entry for entry in entries if _raw_erp_trace_prefilter(ledger_row, entry)]
|
||||
skipped_common_amounts += 1
|
||||
if len(entries) > args.max_candidates_per_row:
|
||||
ranked = [
|
||||
(_raw_erp_trace_prefilter_score(ledger_row, entry), entry)
|
||||
for entry in entries
|
||||
]
|
||||
ranked = [item for item in ranked if item[0] > 0]
|
||||
ranked.sort(key=lambda item: item[0], reverse=True)
|
||||
entries = [entry for _score, entry in ranked[: args.max_candidates_per_row]]
|
||||
|
||||
row_candidates: list[tuple[float, dict[str, Any], dict[str, Any]]] = []
|
||||
for entry in entries:
|
||||
score, candidate = _raw_erp_trace_score(ledger_row, entry)
|
||||
scored_rows += 1
|
||||
if score < args.min_score:
|
||||
continue
|
||||
row_candidates.append((score, entry, candidate))
|
||||
row_candidates.sort(key=lambda item: item[0], reverse=True)
|
||||
for score, entry, candidate in row_candidates[: args.top_per_row]:
|
||||
row_index = int(ledger_row.get("row_index") or 0)
|
||||
matched_case = _matched_case(score, candidate)
|
||||
payload = {
|
||||
"candidate_id": _candidate_identity(
|
||||
args.year,
|
||||
args.source,
|
||||
source_signature,
|
||||
TRACE_LOGIC_VERSION,
|
||||
status_key,
|
||||
group_index,
|
||||
row_index,
|
||||
candidate,
|
||||
),
|
||||
"ledger_row": ledger_row,
|
||||
"erp_entry": entry,
|
||||
"candidate_row": candidate,
|
||||
}
|
||||
candidates.append(
|
||||
{
|
||||
"candidate_key": payload["candidate_id"],
|
||||
"fiscal_year": args.year,
|
||||
"source_mode": args.source,
|
||||
"source_signature": source_signature,
|
||||
"logic_version": TRACE_LOGIC_VERSION,
|
||||
"status_key": status_key,
|
||||
"group_index": group_index,
|
||||
"row_index": row_index,
|
||||
"ledger_date": clean(ledger_row.get("ledger_date")),
|
||||
"voucher_no": clean(ledger_row.get("voucher_no") or group_meta.get("voucher_no")),
|
||||
"ledger_account_name": clean(ledger_row.get("ledger_account_name")),
|
||||
"ledger_vendor": clean(ledger_row.get("ledger_vendor")),
|
||||
"ledger_desc": clean(ledger_row.get("ledger_desc")),
|
||||
"ledger_amount": amount,
|
||||
"erp_draft_no": clean(candidate.get("draft_no")),
|
||||
"erp_confirmed_no": clean(candidate.get("voucher_confirmed_no")),
|
||||
"erp_account_name": clean(candidate.get("voucher_account_name")),
|
||||
"erp_vendor": clean(candidate.get("voucher_vendor")),
|
||||
"erp_desc": clean(candidate.get("voucher_desc")),
|
||||
"erp_amount": max(
|
||||
abs(parse_amount(candidate.get("voucher_debit"))),
|
||||
abs(parse_amount(candidate.get("voucher_credit"))),
|
||||
),
|
||||
"amount_field": clean(entry.get("_raw_amount_field")),
|
||||
"score": float(score),
|
||||
"matched_case": matched_case,
|
||||
"review_reason": matched_case,
|
||||
"candidate_json": json.dumps(payload, ensure_ascii=False, default=str),
|
||||
}
|
||||
)
|
||||
if candidates:
|
||||
_insert_candidates(conn, args.year, args.source, source_signature, candidates)
|
||||
return {
|
||||
"year": args.year,
|
||||
"source_mode": args.source,
|
||||
"source_signature": source_signature,
|
||||
"group_offset": args.group_offset,
|
||||
"limit_groups": args.limit_groups,
|
||||
"groups": len(source_groups),
|
||||
"amount_buckets": len(amount_index),
|
||||
"scored_rows": scored_rows,
|
||||
"common_amount_prefilters": skipped_common_amounts,
|
||||
"inserted_candidates": len(candidates),
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Build bounded raw ERP trace candidate cache for voucher compare review.")
|
||||
parser.add_argument("--year", type=int, required=True)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
choices=("current", "stale-diagnostic"),
|
||||
default="current",
|
||||
help="current는 현재 로직 ready projection만 사용합니다. stale-diagnostic은 진단용 후보 산출에만 사용하세요.",
|
||||
)
|
||||
parser.add_argument("--db", type=Path, default=DB_PATH)
|
||||
parser.add_argument("--reset", action="store_true")
|
||||
parser.add_argument("--limit-groups", type=int, default=0)
|
||||
parser.add_argument("--group-offset", type=int, default=0)
|
||||
parser.add_argument("--ledger-date", action="append", default=[])
|
||||
parser.add_argument("--voucher-no", action="append", default=[])
|
||||
parser.add_argument("--min-score", type=float, default=70)
|
||||
parser.add_argument("--top-per-row", type=int, default=5)
|
||||
parser.add_argument("--max-candidates-per-row", type=int, default=30)
|
||||
parser.add_argument("--date-prefilter-threshold", type=int, default=30)
|
||||
parser.add_argument("--date-window-days", type=int, default=62)
|
||||
parser.add_argument("--prefilter-threshold", type=int, default=30)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(build_candidates(args), ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from main import engine, hash_password, init_db # noqa: E402
|
||||
from sqlalchemy import text # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Create or update an intranet app user.")
|
||||
parser.add_argument("username")
|
||||
parser.add_argument("--password", required=True)
|
||||
parser.add_argument("--display-name", default="")
|
||||
parser.add_argument("--role", choices=["admin", "viewer"], default="viewer")
|
||||
parser.add_argument("--inactive", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
init_db()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO app_users (
|
||||
username, password_hash, display_name, is_active, is_admin, created_at, updated_at
|
||||
) VALUES (
|
||||
:username, :password_hash, :display_name, :is_active, :is_admin,
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||
)
|
||||
ON CONFLICT(username) DO UPDATE SET
|
||||
password_hash = excluded.password_hash,
|
||||
display_name = excluded.display_name,
|
||||
is_active = excluded.is_active,
|
||||
is_admin = excluded.is_admin,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
),
|
||||
{
|
||||
"username": args.username,
|
||||
"password_hash": hash_password(args.password),
|
||||
"display_name": args.display_name or args.username,
|
||||
"is_active": 0 if args.inactive else 1,
|
||||
"is_admin": 1 if args.role == "admin" else 0,
|
||||
},
|
||||
)
|
||||
user_id = conn.execute(
|
||||
text("SELECT id FROM app_users WHERE username = :username"),
|
||||
{"username": args.username},
|
||||
).scalar_one()
|
||||
role_id = conn.execute(
|
||||
text("SELECT id FROM app_roles WHERE role_key = :role_key"),
|
||||
{"role_key": args.role},
|
||||
).scalar_one()
|
||||
conn.execute(text("DELETE FROM app_user_roles WHERE user_id = :user_id"), {"user_id": user_id})
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT OR IGNORE INTO app_user_roles (user_id, role_id, created_at)
|
||||
VALUES (:user_id, :role_id, CURRENT_TIMESTAMP)
|
||||
"""
|
||||
),
|
||||
{"user_id": user_id, "role_id": role_id},
|
||||
)
|
||||
print(f"ok: {args.username} ({args.role})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,397 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
import math
|
||||
import struct
|
||||
import zlib
|
||||
import zipfile
|
||||
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
REPORT_DIR = BASE_DIR / "reports"
|
||||
OUTPUT_PATH = REPORT_DIR / "my-intranet-app_architecture_report_20260522_v3.docx"
|
||||
PAGE_WIDTH = 11906
|
||||
MARGIN = 620
|
||||
CONTENT_WIDTH = PAGE_WIDTH - (MARGIN * 2)
|
||||
|
||||
|
||||
def esc(value: object) -> str:
|
||||
return escape(str(value), {'"': """})
|
||||
|
||||
|
||||
def run(text: str, *, bold: bool = False, size: int | None = None, font: str | None = None) -> str:
|
||||
props: list[str] = []
|
||||
if bold:
|
||||
props.append("<w:b/>")
|
||||
if size:
|
||||
props.append(f'<w:sz w:val="{size}"/><w:szCs w:val="{size}"/>')
|
||||
if font:
|
||||
props.append(f'<w:rFonts w:ascii="{font}" w:hAnsi="{font}" w:eastAsia="{font}" w:cs="{font}"/>')
|
||||
rpr = f"<w:rPr>{''.join(props)}</w:rPr>" if props else ""
|
||||
preserve = ' xml:space="preserve"' if text[:1].isspace() or text[-1:].isspace() else ""
|
||||
return f"<w:r>{rpr}<w:t{preserve}>{esc(text)}</w:t></w:r>"
|
||||
|
||||
|
||||
def para(text: str = "", *, style: str | None = None, bold: bool = False, size: int | None = None, font: str | None = None, after: int | None = None) -> str:
|
||||
pprops: list[str] = []
|
||||
if style:
|
||||
pprops.append(f'<w:pStyle w:val="{style}"/>')
|
||||
if after is not None:
|
||||
pprops.append(f'<w:spacing w:after="{after}"/>')
|
||||
ppr = f"<w:pPr>{''.join(pprops)}</w:pPr>" if pprops else ""
|
||||
return f"<w:p>{ppr}{run(text, bold=bold, size=size, font=font)}</w:p>"
|
||||
|
||||
|
||||
def bullet(text: str) -> str:
|
||||
return para("• " + text, after=60)
|
||||
|
||||
|
||||
def table(headers: list[str], rows: list[list[str]], widths: list[int] | None = None) -> str:
|
||||
if widths is None:
|
||||
widths = [CONTENT_WIDTH // len(headers)] * len(headers)
|
||||
grid = "".join(f'<w:gridCol w:w="{w}"/>' for w in widths)
|
||||
|
||||
def cell(text: str, width: int, header: bool = False) -> str:
|
||||
fill = '<w:shd w:fill="E8EEF7"/>' if header else ""
|
||||
props = (
|
||||
f'<w:tcPr><w:tcW w:w="{width}" w:type="dxa"/>{fill}'
|
||||
'<w:tcMar><w:top w:w="30" w:type="dxa"/><w:left w:w="45" w:type="dxa"/>'
|
||||
'<w:bottom w:w="30" w:type="dxa"/><w:right w:w="45" w:type="dxa"/></w:tcMar></w:tcPr>'
|
||||
)
|
||||
return f"<w:tc>{props}{para(text, bold=header, size=16, after=0)}</w:tc>"
|
||||
|
||||
rows_xml = ["<w:tr>" + "".join(cell(h, widths[i], True) for i, h in enumerate(headers)) + "</w:tr>"]
|
||||
rows_xml.extend(
|
||||
"<w:tr>" + "".join(cell(c, widths[i]) for i, c in enumerate(row)) + "</w:tr>"
|
||||
for row in rows
|
||||
)
|
||||
return (
|
||||
'<w:tbl><w:tblPr><w:tblStyle w:val="TableGrid"/>'
|
||||
f'<w:tblW w:w="{CONTENT_WIDTH}" w:type="dxa"/><w:tblLayout w:type="fixed"/>'
|
||||
"</w:tblPr>"
|
||||
f"<w:tblGrid>{grid}</w:tblGrid>{''.join(rows_xml)}</w:tbl>"
|
||||
)
|
||||
|
||||
|
||||
class PngCanvas:
|
||||
def __init__(self, width: int, height: int, bg: tuple[int, int, int] = (255, 255, 255)):
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.px = bytearray(bg * width * height)
|
||||
|
||||
def set(self, x: int, y: int, color: tuple[int, int, int]) -> None:
|
||||
if 0 <= x < self.width and 0 <= y < self.height:
|
||||
i = (y * self.width + x) * 3
|
||||
self.px[i : i + 3] = bytes(color)
|
||||
|
||||
def rect(self, x: int, y: int, w: int, h: int, fill: tuple[int, int, int], border: tuple[int, int, int], bw: int = 3) -> None:
|
||||
for yy in range(y, y + h):
|
||||
for xx in range(x, x + w):
|
||||
if x <= xx < x + w and y <= yy < y + h:
|
||||
self.set(xx, yy, fill)
|
||||
for n in range(bw):
|
||||
self.line(x + n, y + n, x + w - 1 - n, y + n, border)
|
||||
self.line(x + n, y + h - 1 - n, x + w - 1 - n, y + h - 1 - n, border)
|
||||
self.line(x + n, y + n, x + n, y + h - 1 - n, border)
|
||||
self.line(x + w - 1 - n, y + n, x + w - 1 - n, y + h - 1 - n, border)
|
||||
|
||||
def line(self, x1: int, y1: int, x2: int, y2: int, color: tuple[int, int, int], width: int = 3) -> None:
|
||||
dx = abs(x2 - x1)
|
||||
dy = -abs(y2 - y1)
|
||||
sx = 1 if x1 < x2 else -1
|
||||
sy = 1 if y1 < y2 else -1
|
||||
err = dx + dy
|
||||
x, y = x1, y1
|
||||
while True:
|
||||
r = width // 2
|
||||
for yy in range(y - r, y + r + 1):
|
||||
for xx in range(x - r, x + r + 1):
|
||||
self.set(xx, yy, color)
|
||||
if x == x2 and y == y2:
|
||||
break
|
||||
e2 = 2 * err
|
||||
if e2 >= dy:
|
||||
err += dy
|
||||
x += sx
|
||||
if e2 <= dx:
|
||||
err += dx
|
||||
y += sy
|
||||
|
||||
def arrow(self, x1: int, y1: int, x2: int, y2: int, color: tuple[int, int, int] = (75, 88, 99)) -> None:
|
||||
self.line(x1, y1, x2, y2, color, 4)
|
||||
ang = math.atan2(y2 - y1, x2 - x1)
|
||||
for a in (ang + 2.55, ang - 2.55):
|
||||
self.line(x2, y2, int(x2 + 22 * math.cos(a)), int(y2 + 22 * math.sin(a)), color, 4)
|
||||
|
||||
def digit(self, x: int, y: int, digit: str, color: tuple[int, int, int] = (20, 39, 54), scale: int = 8) -> None:
|
||||
glyphs = {
|
||||
"0": ["111", "101", "101", "101", "111"],
|
||||
"1": ["010", "110", "010", "010", "111"],
|
||||
"2": ["111", "001", "111", "100", "111"],
|
||||
"3": ["111", "001", "111", "001", "111"],
|
||||
"4": ["101", "101", "111", "001", "001"],
|
||||
"5": ["111", "100", "111", "001", "111"],
|
||||
"6": ["111", "100", "111", "101", "111"],
|
||||
"7": ["111", "001", "010", "010", "010"],
|
||||
"8": ["111", "101", "111", "101", "111"],
|
||||
"9": ["111", "101", "111", "001", "111"],
|
||||
}[digit]
|
||||
for gy, row in enumerate(glyphs):
|
||||
for gx, v in enumerate(row):
|
||||
if v == "1":
|
||||
self.rect(x + gx * scale, y + gy * scale, scale - 1, scale - 1, color, color, 1)
|
||||
|
||||
def number_badge(self, x: int, y: int, n: int) -> None:
|
||||
self.rect(x - 28, y - 28, 56, 56, (255, 255, 255), (47, 111, 163), 4)
|
||||
self.digit(x - 12, y - 18, str(n), scale=8)
|
||||
|
||||
def png(self) -> bytes:
|
||||
rows = bytearray()
|
||||
stride = self.width * 3
|
||||
for y in range(self.height):
|
||||
rows.append(0)
|
||||
rows.extend(self.px[y * stride : (y + 1) * stride])
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", self.width, self.height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(bytes(rows), 9))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def architecture_png() -> bytes:
|
||||
c = PngCanvas(1200, 640, (252, 254, 255))
|
||||
blue, green, gold, purple, gray = (232, 242, 252), (237, 248, 237), (255, 248, 230), (246, 239, 250), (78, 91, 104)
|
||||
c.rect(55, 260, 170, 90, blue, (47, 111, 163)); c.number_badge(140, 305, 1)
|
||||
c.rect(310, 215, 245, 180, green, (63, 143, 77)); c.number_badge(432, 305, 2)
|
||||
c.rect(650, 45, 230, 90, gold, (176, 122, 26)); c.number_badge(765, 90, 3)
|
||||
c.rect(650, 185, 230, 105, blue, (47, 111, 163)); c.number_badge(765, 238, 4)
|
||||
c.rect(650, 340, 230, 105, purple, (127, 85, 160)); c.number_badge(765, 393, 5)
|
||||
c.rect(650, 500, 230, 90, gold, (176, 122, 26)); c.number_badge(765, 545, 6)
|
||||
c.rect(970, 230, 175, 145, green, (63, 143, 77)); c.number_badge(1058, 303, 7)
|
||||
c.arrow(225, 305, 310, 305, gray); c.arrow(555, 270, 650, 92, gray); c.arrow(555, 305, 650, 238, gray)
|
||||
c.arrow(555, 335, 650, 393, gray); c.arrow(555, 375, 650, 545, gray); c.arrow(880, 238, 970, 285, gray); c.arrow(880, 393, 970, 325, gray)
|
||||
return c.png()
|
||||
|
||||
|
||||
def flow_png() -> bytes:
|
||||
c = PngCanvas(1200, 640, (255, 255, 255))
|
||||
colors = [(232, 242, 252), (237, 248, 237), (255, 248, 230), (246, 239, 250), (238, 242, 246)]
|
||||
border = [(47, 111, 163), (63, 143, 77), (176, 122, 26), (127, 85, 160), (90, 105, 120)]
|
||||
boxes = [(60, 90, 190, 105), (330, 90, 190, 105), (600, 90, 190, 105), (870, 90, 190, 105),
|
||||
(330, 350, 190, 105), (600, 350, 190, 105), (870, 350, 190, 105)]
|
||||
for i, (x, y, w, h) in enumerate(boxes, start=1):
|
||||
c.rect(x, y, w, h, colors[(i - 1) % len(colors)], border[(i - 1) % len(border)])
|
||||
c.number_badge(x + w // 2, y + h // 2, i)
|
||||
gray = (78, 91, 104)
|
||||
c.arrow(250, 142, 330, 142, gray); c.arrow(520, 142, 600, 142, gray); c.arrow(790, 142, 870, 142, gray)
|
||||
c.arrow(965, 195, 965, 350, gray); c.arrow(870, 402, 790, 402, gray); c.arrow(600, 402, 520, 402, gray)
|
||||
c.arrow(425, 350, 425, 195, gray)
|
||||
return c.png()
|
||||
|
||||
|
||||
def image_paragraph(rel_id: str, width_emu: int = 6_950_000, height_emu: int = 3_700_000) -> str:
|
||||
return f"""
|
||||
<w:p><w:pPr><w:spacing w:after="100"/></w:pPr><w:r><w:drawing>
|
||||
<wp:inline distT="0" distB="0" distL="0" distR="0" xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing">
|
||||
<wp:extent cx="{width_emu}" cy="{height_emu}"/><wp:docPr id="{rel_id[3:]}" name="{rel_id}.png"/>
|
||||
<a:graphic xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/picture">
|
||||
<pic:pic xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture">
|
||||
<pic:nvPicPr><pic:cNvPr id="0" name="{rel_id}.png"/><pic:cNvPicPr/></pic:nvPicPr>
|
||||
<pic:blipFill><a:blip r:embed="{rel_id}" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"/><a:stretch><a:fillRect/></a:stretch></pic:blipFill>
|
||||
<pic:spPr><a:xfrm><a:off x="0" y="0"/><a:ext cx="{width_emu}" cy="{height_emu}"/></a:xfrm><a:prstGeom prst="rect"><a:avLst/></a:prstGeom></pic:spPr>
|
||||
</pic:pic></a:graphicData></a:graphic></wp:inline></w:drawing></w:r></w:p>
|
||||
"""
|
||||
|
||||
|
||||
def body_xml() -> str:
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
p: list[str] = []
|
||||
p.append(para("my-intranet-app 아키텍처 분석 보고서", style="Title"))
|
||||
p.append(para(f"작성일: {now} / 기준 경로: {BASE_DIR}", style="Subtitle"))
|
||||
p.append(para("본 보고서의 주 목적은 현재 my-intranet-app의 아키텍처, 데이터 흐름, 주요 결합도와 개선 포인트를 분석하는 것입니다. WSL 제로 세팅 시 보존 범위는 마지막 운영 고려사항으로 덧붙였습니다."))
|
||||
|
||||
p.append(para("1. 분석 결론", style="Heading1"))
|
||||
p.append(bullet("현재 시스템은 FastAPI 단일 애플리케이션 안에 화면 렌더링, JSON API, SQLite 접근, 캐시, 백그라운드 작업, 외부 ERP/WEHAGO 연동이 결합된 내부 업무 앱입니다."))
|
||||
p.append(bullet("업무 데이터의 중심은 SQLite data.db이며, 프로젝트/회계/전표비교/캐시/작업 이력이 같은 DB에 함께 저장됩니다."))
|
||||
p.append(bullet("구조 안정성 측면의 가장 큰 리스크는 main.py의 과도한 책임 집중과 대용량 SQLite 파일에 운영 데이터와 캐시가 공존하는 점입니다."))
|
||||
p.append(bullet("속도는 런타임 캐시, system_page_cache, WEHAGO query/export projection, background job으로 보완하고 있으나, 장기적으로는 DB 슬림화와 모듈 분리가 필요합니다."))
|
||||
|
||||
p.append(para("2. 아키텍처 개요 이미지", style="Heading1"))
|
||||
p.append(image_paragraph("rId10"))
|
||||
p.append(table(
|
||||
["번호", "구성요소", "설명"],
|
||||
[
|
||||
["1", "Browser/UI", "Jinja2로 내려받은 HTML과 브라우저 fetch API가 화면 갱신을 담당"],
|
||||
["2", "FastAPI Runtime(main.py)", "라우트, HTML 렌더링, JSON API, DB 초기화, 캐시, 작업 큐 진입점"],
|
||||
["3", "Templates", "dashboard/projects/process_cost/wehago_compare 등 서버 렌더링 화면"],
|
||||
["4", "SQLite data.db", "업무 데이터, 설정, 캐시, 작업 상태의 중심 저장소"],
|
||||
["5", "WEHAGO Compare", "전표 비교 전문 로직. 파일 정규화, 매칭, 리뷰, 추천, export"],
|
||||
["6", "Hanmac External", "pymysql 기반 외부 ERP DB 조회 및 집계"],
|
||||
["7", "Workers/Jobs", "캐시 재생성, snapshot, export, 유지보수 작업"],
|
||||
],
|
||||
[750, 2500, CONTENT_WIDTH - 3250],
|
||||
))
|
||||
|
||||
p.append(para("3. 런타임/기술 스택", style="Heading1"))
|
||||
p.append(table(
|
||||
["영역", "현재 구성", "역할/관찰"],
|
||||
[
|
||||
["Web", "FastAPI, Uvicorn", "ASGI 기반 단일 서버. HTML과 JSON API를 같은 앱에서 제공"],
|
||||
["UI", "Jinja2, CSS, Vanilla JS", "템플릿별 inline CSS/JS가 많고 fetch 기반 동적 로딩을 사용"],
|
||||
["DB", "SQLite, SQLAlchemy", "로컬 data.db를 중심으로 업무/캐시/작업 데이터 저장"],
|
||||
["Excel", "openpyxl", "업로드 파일 파싱, WEHAGO 상태별 xlsx 내보내기"],
|
||||
["External DB", "pymysql", "Hanmac 외부 MySQL 접속/preview/aggregate"],
|
||||
["DB Browser", "Datasette", "/db 및 /db-browser에서 내부 DB 조회 지원"],
|
||||
],
|
||||
[1400, 2600, CONTENT_WIDTH - 4000],
|
||||
))
|
||||
|
||||
p.append(para("4. 코드 구조 분석", style="Heading1"))
|
||||
p.append(table(
|
||||
["파일/디렉터리", "아키텍처상 책임", "개선 관점"],
|
||||
[
|
||||
["main.py", "FastAPI app 생성, 라우트, DB 초기화, 화면별 bootstrap, 저장 API, system job, Hanmac 연동", "routers/services/repositories/jobs로 단계 분리 필요"],
|
||||
["wehago_compare.py", "WEHAGO/ERP 전표 비교 도메인. 매칭, 리뷰, 추천, projection/cache/export", "비교 도메인으로 분리된 점은 좋으나 내부 함수가 매우 크고 캐시 책임도 함께 큼"],
|
||||
["templates/*.html", "서버 렌더링 화면과 화면별 대형 JS/CSS", "공통 fetch/job polling/table rendering을 static 모듈로 분리 가능"],
|
||||
["scripts/", "서버 실행, WEHAGO 수집/검증/보정, Windows portproxy", "운영 자동화와 일회성 보정 스크립트 구분 필요"],
|
||||
["data.db", "업무 데이터와 캐시/작업 이력 저장", "운영 데이터와 재생성 캐시 분리 또는 보존 정책 필요"],
|
||||
["backups/", "수동/시점 백업", "복구 가치 기준으로 최신/중요 백업만 관리 권장"],
|
||||
],
|
||||
[2000, 4300, CONTENT_WIDTH - 6300],
|
||||
))
|
||||
|
||||
p.append(para("5. 주요 화면/API 경계", style="Heading1"))
|
||||
p.append(table(
|
||||
["화면/도메인", "대표 라우트", "핵심 데이터 흐름"],
|
||||
[
|
||||
["Dashboard", "/, /bootstrap-data, /dashboard/api/rebuild-cache", "transactions/project 집계 -> bootstrap/cache -> 차트/KPI"],
|
||||
["Projects", "/projects, /projects/bootstrap-data, /projects/save-json", "project_* 조회/저장 -> 미계약/관련 프로젝트/비교 상세 API"],
|
||||
["Process Cost", "/process-cost, /process-cost/bootstrap-data", "Hanmac/WEHAGO 소스 선택 -> 프로젝트별 수익/비용/진척/비율 계산"],
|
||||
["Annual Summary", "/annual-summary, /annual-summary/bootstrap-data", "연도/월별 회계 집계 -> 차트 데이터"],
|
||||
["WEHAGO Compare", "/wehago-compare/api/*", "원천 rows -> 비교 결과 -> 상태별 상세 -> 리뷰/매칭/export"],
|
||||
["Hanmac Browser", "/hanmac-browser/api/*", "외부 MySQL 조회 -> preview/aggregate cache -> CSV export"],
|
||||
["System Jobs", "/api/system-jobs/*", "무거운 cache rebuild/export 작업 생성 및 진행률 조회"],
|
||||
],
|
||||
[1900, 3300, CONTENT_WIDTH - 5200],
|
||||
))
|
||||
|
||||
p.append(para("6. 데이터 흐름 이미지", style="Heading1"))
|
||||
p.append(image_paragraph("rId11"))
|
||||
p.append(table(
|
||||
["번호", "흐름 단계", "설명"],
|
||||
[
|
||||
["1", "원천 데이터", "Excel 업로드, WEHAGO_DB 파일, Hanmac 외부 DB, 사용자 입력"],
|
||||
["2", "수집/정규화", "main.py와 wehago_compare.py에서 날짜/금액/전표번호/프로젝트코드 정규화"],
|
||||
["3", "영속 저장", "transactions, project_*, wehago_* 테이블에 저장"],
|
||||
["4", "집계/비교 계산", "프로젝트 원가, 연도 집계, 전표 매칭, 상태별 metric 계산"],
|
||||
["5", "캐시/작업", "system_page_cache, wehago query cache, background jobs로 무거운 조회 완화"],
|
||||
["6", "API 응답", "bootstrap-data 및 상세 JSON API로 화면에 전달"],
|
||||
["7", "화면 표시/export", "Jinja2 화면, fetch 갱신, xlsx/csv 다운로드"],
|
||||
],
|
||||
[750, 2200, CONTENT_WIDTH - 2950],
|
||||
))
|
||||
|
||||
p.append(para("7. 데이터 아키텍처", style="Heading1"))
|
||||
p.append(table(
|
||||
["테이블 그룹", "대표 테이블", "아키텍처 의미"],
|
||||
[
|
||||
["업무 원장", "transactions", "회계 전표/거래 행의 중심 원천"],
|
||||
["프로젝트", "project_basic_info, project_status, project_contract_info, project_billing_entries, project_collection_entries", "프로젝트 기본/계약/청구/수금/상태"],
|
||||
["프로젝트 분석", "project_exec_budget_entries, project_actual_input_entries, project_task_plan_entries, project_analysis_settings", "원가/투입/계획/분석 설정"],
|
||||
["WEHAGO 비교 원천", "wehago_source_files, wehago_voucher_rows, wehago_ledger_rows", "ERP 전표와 WEHAGO 원장 정규화 데이터"],
|
||||
["WEHAGO 비교 결과", "wehago_comparison_results, wehago_recheck_reviews, wehago_manual_pair_matches", "비교 결과와 사용자가 만든 검토/매칭 상태"],
|
||||
["캐시/작업", "system_page_cache, system_jobs, wehago_*_cache, hanmac_*_cache", "속도 보완용. 일부는 재생성 가능"],
|
||||
["설정/운영", "app_option_items, app_keyword_rules, hanmac_holidays, db_backup_history", "분류 규칙, 옵션, 휴일, 백업 이력"],
|
||||
],
|
||||
[1800, 4300, CONTENT_WIDTH - 6100],
|
||||
))
|
||||
|
||||
p.append(para("8. 구조 안정성/속도 개선 포인트", style="Heading1"))
|
||||
p.append(table(
|
||||
["개선 영역", "현재 리스크", "권장 방향"],
|
||||
[
|
||||
["main.py 책임 분리", "라우트/DB/worker/비즈니스 로직 집중", "도메인별 router, service, repository, job 모듈로 점진 분리"],
|
||||
["DB 관리", "14GB 수준 SQLite에 운영 데이터와 캐시 공존", "캐시 보존 정책, VACUUM/ANALYZE, cache DB 분리 검토"],
|
||||
["WEHAGO 비교", "projection/cache가 많고 상태별 경로가 복잡", "상태별 query path 정리, 캐시 키 문서화, 재계산 CLI 표준화"],
|
||||
["Frontend", "템플릿별 inline JS/CSS가 큼", "공통 fetch/polling/render 유틸을 static JS/CSS로 이동"],
|
||||
["작업 큐", "DB 테이블 기반 작업 상태와 런타임 worker 결합", "작업 타입/상태 전이 규칙 문서화 및 stale job 정리 강화"],
|
||||
["테스트", "구조 변경 후 회귀 확인 경로 부족", "핵심 bootstrap API와 저장 API smoke test 추가"],
|
||||
],
|
||||
[1800, 3300, CONTENT_WIDTH - 5100],
|
||||
))
|
||||
|
||||
p.append(para("9. WSL 제로 세팅 시 보존 범위(부가 운영 고려사항)", style="Heading1"))
|
||||
p.append(para("이 절은 이관 방법 보고서가 아니라, 현재 아키텍처를 보존 가능한 상태로 유지하려면 어떤 정보를 어느 수준까지 관리해야 하는지에 대한 부가 판단입니다."))
|
||||
p.append(table(
|
||||
["대상", "보존 수준", "이유"],
|
||||
[
|
||||
["코드", "필수", "main.py, wehago_compare.py, templates, scripts, requirements는 앱 동작의 본체"],
|
||||
["SQLite DB", "필수", "업무 데이터와 사용자 검토/매칭/설정이 data.db에 존재. 코드만으로 복구 불가"],
|
||||
["WAL/SHM", "조건부 필수", "실행 중 복사라면 data.db-wal 변경분 누락 위험. 서버 중지 또는 checkpoint/backup 필요"],
|
||||
["DB dump", "필수 또는 강력 권장", "새 환경 복원 검증용. 현재 dump.sql은 0 bytes라 유효하지 않음"],
|
||||
["원천 Excel/WEHAGO_DB", "강력 권장", "재검증/재처리/비교 로직 개선 시 기준 자료"],
|
||||
["사용자 검토/수동매칭", "필수", "wehago_recheck_reviews, wehago_manual_pair_matches 등은 재생성 어려움"],
|
||||
["캐시 테이블", "선택", "속도에는 도움되지만 구조 개선 후 재생성 가능. DB 슬림화 대상"],
|
||||
["backups", "선별", "최신 정상본과 구조 변경 직전본 위주로 보존"],
|
||||
[".venv/__pycache__", "불필요", "새 WSL에서 재생성"],
|
||||
],
|
||||
[2200, 1600, CONTENT_WIDTH - 3800],
|
||||
))
|
||||
|
||||
p.append(para("10. 최종 권고", style="Heading1"))
|
||||
p.append(bullet("아키텍처 개선의 1순위는 기능 추가보다 책임 분리와 DB/캐시 관리 기준 정립입니다."))
|
||||
p.append(bullet("WSL 제로 세팅을 하더라도 목표는 '코드 이관'이 아니라 '동일 업무 상태를 복원 가능한 형태로 보존'하는 것입니다."))
|
||||
p.append(bullet("구조 개선 시작 전에는 유효한 SQLite 백업/dump를 새로 만들고, 원천파일과 사용자 검토 데이터의 보존 여부를 반드시 확인해야 합니다."))
|
||||
|
||||
p.append(para("Appendix. 관찰된 현재 상태", style="Heading1"))
|
||||
p.append(bullet("data.db 약 14GB, data.db-wal 약 183MB, dump.sql 0 bytes 상태를 확인했습니다."))
|
||||
p.append(bullet("현재 git working tree에는 기존 수정 파일과 미추적 파일이 존재합니다. 구조 변경 전 기준점을 별도로 고정하는 것이 좋습니다."))
|
||||
p.append(bullet("검토 파일: requirements.txt, main.py, wehago_compare.py, templates/*.html, data.db sqlite_master schema."))
|
||||
|
||||
sect = f'<w:sectPr><w:pgSz w:w="{PAGE_WIDTH}" w:h="16838"/><w:pgMar w:top="720" w:right="{MARGIN}" w:bottom="720" w:left="{MARGIN}" w:header="360" w:footer="360" w:gutter="0"/></w:sectPr>'
|
||||
return "".join(p) + sect
|
||||
|
||||
|
||||
def styles_xml() -> str:
|
||||
return """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
|
||||
<w:style w:type="paragraph" w:default="1" w:styleId="Normal"><w:name w:val="Normal"/><w:qFormat/><w:rPr><w:rFonts w:ascii="Arial" w:hAnsi="Arial" w:eastAsia="Malgun Gothic"/><w:sz w:val="20"/></w:rPr><w:pPr><w:spacing w:after="90" w:line="246" w:lineRule="auto"/></w:pPr></w:style>
|
||||
<w:style w:type="paragraph" w:styleId="Title"><w:name w:val="Title"/><w:basedOn w:val="Normal"/><w:qFormat/><w:rPr><w:b/><w:sz w:val="34"/></w:rPr><w:pPr><w:spacing w:after="200"/></w:pPr></w:style>
|
||||
<w:style w:type="paragraph" w:styleId="Subtitle"><w:name w:val="Subtitle"/><w:basedOn w:val="Normal"/><w:qFormat/><w:rPr><w:color w:val="5B6770"/><w:sz w:val="18"/></w:rPr></w:style>
|
||||
<w:style w:type="paragraph" w:styleId="Heading1"><w:name w:val="heading 1"/><w:basedOn w:val="Normal"/><w:qFormat/><w:rPr><w:b/><w:color w:val="1F4E79"/><w:sz w:val="25"/></w:rPr><w:pPr><w:spacing w:before="250" w:after="100"/><w:outlineLvl w:val="0"/></w:pPr></w:style>
|
||||
<w:style w:type="table" w:styleId="TableGrid"><w:name w:val="Table Grid"/><w:basedOn w:val="TableNormal"/><w:qFormat/><w:tblPr><w:tblBorders><w:top w:val="single" w:sz="4" w:color="AAB7C4"/><w:left w:val="single" w:sz="4" w:color="AAB7C4"/><w:bottom w:val="single" w:sz="4" w:color="AAB7C4"/><w:right w:val="single" w:sz="4" w:color="AAB7C4"/><w:insideH w:val="single" w:sz="4" w:color="D5DDE6"/><w:insideV w:val="single" w:sz="4" w:color="D5DDE6"/></w:tblBorders></w:tblPr></w:style>
|
||||
</w:styles>"""
|
||||
|
||||
|
||||
def write_docx() -> None:
|
||||
created = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
document = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><w:body>{body_xml()}</w:body></w:document>"""
|
||||
files = {
|
||||
"[Content_Types].xml": """<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml"/><Override PartName="/word/settings.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml"/><Override PartName="/docProps/core.xml" ContentType="application/vnd.openxmlformats-package.core-properties+xml"/><Override PartName="/docProps/app.xml" ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/></Types>""",
|
||||
"_rels/.rels": """<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/><Relationship Id="rId3" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties" Target="docProps/app.xml"/></Relationships>""",
|
||||
"word/_rels/document.xml.rels": """<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/settings" Target="settings.xml"/><Relationship Id="rId10" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" Target="media/architecture.png"/><Relationship Id="rId11" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" Target="media/dataflow.png"/></Relationships>""",
|
||||
"word/document.xml": document,
|
||||
"word/styles.xml": styles_xml(),
|
||||
"word/settings.xml": """<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:settings xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:zoom w:percent="100"/></w:settings>""",
|
||||
"word/media/architecture.png": architecture_png(),
|
||||
"word/media/dataflow.png": flow_png(),
|
||||
"docProps/core.xml": f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?><cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><dc:title>my-intranet-app 아키텍처 분석 보고서</dc:title><dc:creator>Codex</dc:creator><cp:lastModifiedBy>Codex</cp:lastModifiedBy><dcterms:created xsi:type="dcterms:W3CDTF">{created}</dcterms:created><dcterms:modified xsi:type="dcterms:W3CDTF">{created}</dcterms:modified></cp:coreProperties>""",
|
||||
"docProps/app.xml": """<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Properties xmlns="http://schemas.openxmlformats.org/officeDocument/2006/extended-properties"><Application>Codex OOXML Generator</Application></Properties>""",
|
||||
}
|
||||
REPORT_DIR.mkdir(exist_ok=True)
|
||||
with zipfile.ZipFile(OUTPUT_PATH, "w", compression=zipfile.ZIP_DEFLATED) as docx:
|
||||
for name, content in files.items():
|
||||
docx.writestr(name, content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
write_docx()
|
||||
print(OUTPUT_PATH)
|
||||
@@ -0,0 +1,6 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$projectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
|
||||
docker compose -f compose.yaml -f compose.dev.yaml down
|
||||
@@ -0,0 +1,65 @@
|
||||
param(
|
||||
[switch]$Build
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$projectRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
|
||||
$imageInputFiles = @(
|
||||
"Dockerfile",
|
||||
"requirements.txt",
|
||||
".dockerignore",
|
||||
"compose.yaml",
|
||||
"compose.dev.yaml"
|
||||
)
|
||||
$fingerprintLines = foreach ($relativePath in $imageInputFiles) {
|
||||
$filePath = Join-Path $projectRoot $relativePath
|
||||
if (Test-Path -LiteralPath $filePath) {
|
||||
"$relativePath=$((Get-FileHash -Algorithm SHA256 -LiteralPath $filePath).Hash)"
|
||||
} else {
|
||||
"$relativePath=MISSING"
|
||||
}
|
||||
}
|
||||
$fingerprintText = $fingerprintLines -join "`n"
|
||||
$hashAlgorithm = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$fingerprintBytes = [System.Text.Encoding]::UTF8.GetBytes($fingerprintText)
|
||||
$currentFingerprint = ([System.BitConverter]::ToString($hashAlgorithm.ComputeHash($fingerprintBytes))).Replace("-", "").ToLowerInvariant()
|
||||
} finally {
|
||||
$hashAlgorithm.Dispose()
|
||||
}
|
||||
$stateDir = Join-Path $projectRoot ".dev-state"
|
||||
$stateFile = Join-Path $stateDir "docker-image-inputs.sha256"
|
||||
$previousFingerprint = if (Test-Path -LiteralPath $stateFile) {
|
||||
(Get-Content -LiteralPath $stateFile -Raw).Trim()
|
||||
} else {
|
||||
""
|
||||
}
|
||||
$inputsChanged = -not $previousFingerprint -or $previousFingerprint -ne $currentFingerprint
|
||||
$shouldBuild = $Build -or $inputsChanged
|
||||
|
||||
$composeArgs = @("-f", "compose.yaml", "-f", "compose.dev.yaml", "up", "-d")
|
||||
if ($shouldBuild) {
|
||||
$composeArgs += "--build"
|
||||
}
|
||||
|
||||
if ($Build) {
|
||||
Write-Host "Image rebuild requested explicitly with -Build."
|
||||
} elseif ($inputsChanged) {
|
||||
Write-Host "Docker image inputs changed or were not recorded yet; rebuilding automatically."
|
||||
} else {
|
||||
Write-Host "Docker image inputs are unchanged; starting without rebuild."
|
||||
}
|
||||
|
||||
docker compose @composeArgs
|
||||
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
||||
Set-Content -LiteralPath $stateFile -Value $currentFingerprint -NoNewline
|
||||
docker compose -f compose.yaml -f compose.dev.yaml ps
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Development server: http://127.0.0.1:8010"
|
||||
Write-Host "Source edits reload automatically."
|
||||
Write-Host "Dependency and Docker image-setting changes are detected and rebuilt automatically."
|
||||
Write-Host "Use -Build only when you intentionally want to force a clean image check."
|
||||
@@ -24,9 +24,9 @@ from wehago_compare import (
|
||||
rebuild_comparison_results,
|
||||
upsert_source_file,
|
||||
)
|
||||
from runtime_config import DB_PATH
|
||||
|
||||
|
||||
DB_PATH = BASE_DIR / "data.db"
|
||||
ENGINE = create_engine(f"sqlite:///{DB_PATH}", connect_args={"check_same_thread": False})
|
||||
|
||||
LEDGER_FILES = {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,901 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from runtime_config import DB_PATH
|
||||
from wehago_compare import (
|
||||
QUERY_PROJECTION_VERSION,
|
||||
_account_base_names_compatible,
|
||||
_contained_core_desc_match,
|
||||
_erp_section_identity,
|
||||
_is_obvious_recheck_group,
|
||||
_recheck_group_rank_key,
|
||||
_section_vat_exception_capacity,
|
||||
_short_core_desc_fuzzy_match,
|
||||
_same_or_similar_desc,
|
||||
_wehago_section_identity,
|
||||
clean,
|
||||
)
|
||||
|
||||
|
||||
TARGET_START_YEAR = 2025
|
||||
TARGET_END_YEAR = 2025
|
||||
|
||||
|
||||
GROUP_COLUMNS = (
|
||||
"start_year",
|
||||
"end_year",
|
||||
"status_key",
|
||||
"signature",
|
||||
"group_index",
|
||||
"fiscal_year",
|
||||
"ledger_date",
|
||||
"proof_date",
|
||||
"voucher_no",
|
||||
"draft_no",
|
||||
"ledger_row_count",
|
||||
"voucher_row_count",
|
||||
"ledger_debit",
|
||||
"ledger_credit",
|
||||
"voucher_debit",
|
||||
"voucher_credit",
|
||||
"ledger_accounts",
|
||||
"voucher_accounts",
|
||||
"ledger_vendors",
|
||||
"voucher_vendors",
|
||||
"review_reason",
|
||||
"search_text",
|
||||
)
|
||||
|
||||
ROW_COLUMNS = (
|
||||
"start_year",
|
||||
"end_year",
|
||||
"status_key",
|
||||
"signature",
|
||||
"group_index",
|
||||
"row_index",
|
||||
"fiscal_year",
|
||||
"status_label",
|
||||
"ledger_date",
|
||||
"proof_date",
|
||||
"voucher_no",
|
||||
"draft_no",
|
||||
"ledger_account_name",
|
||||
"voucher_account_name",
|
||||
"ledger_vendor",
|
||||
"voucher_vendor",
|
||||
"ledger_debit",
|
||||
"ledger_credit",
|
||||
"voucher_debit",
|
||||
"voucher_credit",
|
||||
"ledger_desc",
|
||||
"voucher_desc",
|
||||
"review_reason",
|
||||
"matched_case",
|
||||
"ledger_row_key",
|
||||
"voucher_row_key",
|
||||
"match_identity_key",
|
||||
)
|
||||
|
||||
|
||||
def _dict(row: sqlite3.Row) -> dict[str, Any]:
|
||||
return {key: row[key] for key in row.keys()}
|
||||
|
||||
|
||||
def _amount_key(value: Any) -> str:
|
||||
try:
|
||||
amount = float(str(value or "0").replace(",", ""))
|
||||
except Exception:
|
||||
amount = 0.0
|
||||
if abs(amount - round(amount)) < 0.0001:
|
||||
return str(int(round(amount)))
|
||||
return f"{amount:.2f}".rstrip("0").rstrip(".")
|
||||
|
||||
|
||||
def _parse_amount(value: Any) -> float:
|
||||
try:
|
||||
return float(str(value or "0").replace(",", ""))
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _has_ledger_value(row: dict[str, Any]) -> bool:
|
||||
return bool(clean(row.get("ledger_account_name"))) and (
|
||||
abs(_parse_amount(row.get("ledger_debit"))) > 0.0001
|
||||
or abs(_parse_amount(row.get("ledger_credit"))) > 0.0001
|
||||
)
|
||||
|
||||
|
||||
def _has_voucher_value(row: dict[str, Any]) -> bool:
|
||||
return bool(clean(row.get("voucher_account_name"))) and (
|
||||
abs(_parse_amount(row.get("voucher_debit"))) > 0.0001
|
||||
or abs(_parse_amount(row.get("voucher_credit"))) > 0.0001
|
||||
)
|
||||
|
||||
|
||||
def _same_side_amount_match(ledger_row: dict[str, Any], voucher_row: dict[str, Any]) -> bool:
|
||||
return (
|
||||
abs(_parse_amount(ledger_row.get("ledger_debit")) - _parse_amount(voucher_row.get("voucher_debit"))) < 0.5
|
||||
and abs(_parse_amount(ledger_row.get("ledger_credit")) - _parse_amount(voucher_row.get("voucher_credit"))) < 0.5
|
||||
and (
|
||||
abs(_parse_amount(ledger_row.get("ledger_debit"))) > 0.0001
|
||||
or abs(_parse_amount(ledger_row.get("ledger_credit"))) > 0.0001
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _desc_core_match(ledger_row: dict[str, Any], voucher_row: dict[str, Any]) -> bool:
|
||||
probe = {
|
||||
"ledger_desc": ledger_row.get("ledger_desc"),
|
||||
"voucher_desc": voucher_row.get("voucher_desc"),
|
||||
}
|
||||
matched = (
|
||||
_same_or_similar_desc(probe)
|
||||
or _contained_core_desc_match(ledger_row.get("ledger_desc"), voucher_row.get("voucher_desc"))
|
||||
or _short_core_desc_fuzzy_match(ledger_row.get("ledger_desc"), voucher_row.get("voucher_desc"))
|
||||
)
|
||||
if matched:
|
||||
return True
|
||||
|
||||
stop_words = {
|
||||
"관련",
|
||||
"전표",
|
||||
"처리",
|
||||
"정산",
|
||||
"금액",
|
||||
"비용",
|
||||
"지급",
|
||||
"입금",
|
||||
"출금",
|
||||
"매입",
|
||||
"매출",
|
||||
"급여",
|
||||
"제경비",
|
||||
}
|
||||
|
||||
def tokens(value: Any) -> set[str]:
|
||||
found: set[str] = set()
|
||||
for token in re.split(r"[^0-9A-Za-z가-힣]+", clean(value)):
|
||||
token = token.strip()
|
||||
if len(token) < 2 or token in stop_words or re.fullmatch(r"\d+월?", token):
|
||||
continue
|
||||
found.add(token)
|
||||
return found
|
||||
|
||||
shared = tokens(ledger_row.get("ledger_desc")) & tokens(voucher_row.get("voucher_desc"))
|
||||
return len(shared) >= 2 or any(len(token) >= 3 for token in shared)
|
||||
|
||||
|
||||
def _merge_internal_recheck_pairs(group: dict[str, Any]) -> dict[str, Any]:
|
||||
rows = [dict(row) for row in group.get("rows") or []]
|
||||
ledger_only = [row for row in rows if _has_ledger_value(row) and not _has_voucher_value(row)]
|
||||
voucher_only = [row for row in rows if _has_voucher_value(row) and not _has_ledger_value(row)]
|
||||
if not ledger_only or not voucher_only:
|
||||
return group
|
||||
|
||||
used_ledger: set[int] = set()
|
||||
used_voucher: set[int] = set()
|
||||
merged_rows: list[dict[str, Any]] = []
|
||||
candidates: list[tuple[float, int, int]] = []
|
||||
for ledger_index, ledger_row in enumerate(ledger_only):
|
||||
for voucher_index, voucher_row in enumerate(voucher_only):
|
||||
if not _same_side_amount_match(ledger_row, voucher_row):
|
||||
continue
|
||||
if not _account_base_names_compatible(ledger_row.get("ledger_account_name"), voucher_row.get("voucher_account_name")):
|
||||
continue
|
||||
if not _desc_core_match(ledger_row, voucher_row):
|
||||
continue
|
||||
amount = max(abs(_parse_amount(ledger_row.get("ledger_debit"))), abs(_parse_amount(ledger_row.get("ledger_credit"))))
|
||||
candidates.append((amount, ledger_index, voucher_index))
|
||||
for _amount, ledger_index, voucher_index in sorted(candidates, reverse=True):
|
||||
if ledger_index in used_ledger or voucher_index in used_voucher:
|
||||
continue
|
||||
ledger_row = ledger_only[ledger_index]
|
||||
voucher_row = voucher_only[voucher_index]
|
||||
merged = dict(ledger_row)
|
||||
for field in (
|
||||
"proof_date",
|
||||
"draft_no",
|
||||
"voucher_account_name",
|
||||
"voucher_vendor",
|
||||
"voucher_debit",
|
||||
"voucher_credit",
|
||||
"voucher_desc",
|
||||
"voucher_row_key",
|
||||
"match_identity_key",
|
||||
):
|
||||
merged[field] = voucher_row.get(field, "")
|
||||
merged["review_reason"] = "RECHECK_INTERNAL_CORE_MATCH"
|
||||
merged_rows.append(merged)
|
||||
used_ledger.add(ledger_index)
|
||||
used_voucher.add(voucher_index)
|
||||
if not merged_rows:
|
||||
return group
|
||||
|
||||
remaining_rows: list[dict[str, Any]] = []
|
||||
ledger_ids = {id(row): index for index, row in enumerate(ledger_only)}
|
||||
voucher_ids = {id(row): index for index, row in enumerate(voucher_only)}
|
||||
for row in rows:
|
||||
if _has_ledger_value(row) and not _has_voucher_value(row):
|
||||
index = ledger_ids.get(id(row))
|
||||
if index is not None and index in used_ledger:
|
||||
continue
|
||||
if _has_voucher_value(row) and not _has_ledger_value(row):
|
||||
index = voucher_ids.get(id(row))
|
||||
if index is not None and index in used_voucher:
|
||||
continue
|
||||
remaining_rows.append(row)
|
||||
return {**group, "rows": merged_rows + remaining_rows}
|
||||
|
||||
|
||||
def _group_manual_review_key(group: dict[str, Any]) -> str:
|
||||
summary = group.get("summary") or {}
|
||||
return "|".join(
|
||||
[
|
||||
"manual-recheck",
|
||||
clean(summary.get("fiscal_year")),
|
||||
clean(summary.get("ledger_date")),
|
||||
clean(summary.get("voucher_no")),
|
||||
clean(summary.get("draft_no")),
|
||||
"",
|
||||
"",
|
||||
_amount_key(summary.get("ledger_debit")),
|
||||
_amount_key(summary.get("ledger_credit")),
|
||||
_amount_key(summary.get("voucher_debit")),
|
||||
_amount_key(summary.get("voucher_credit")),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _row_change_key(row: dict[str, Any], change_type: str) -> str:
|
||||
explicit_key = clean(row.get("review_key")) or clean(row.get("match_identity_key"))
|
||||
if explicit_key:
|
||||
return f"{change_type}:{explicit_key}"
|
||||
return "|".join(
|
||||
[
|
||||
change_type,
|
||||
clean(row.get("fiscal_year")),
|
||||
clean(row.get("ledger_date")),
|
||||
clean(row.get("voucher_no")),
|
||||
clean(row.get("draft_no")),
|
||||
clean(row.get("ledger_account_name")),
|
||||
clean(row.get("voucher_account_name")),
|
||||
_amount_key(row.get("ledger_debit")),
|
||||
_amount_key(row.get("ledger_credit")),
|
||||
_amount_key(row.get("voucher_debit")),
|
||||
_amount_key(row.get("voucher_credit")),
|
||||
clean(row.get("ledger_desc")),
|
||||
clean(row.get("voucher_desc")),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _ensure_change_table(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS wehago_recheck_row_changes (
|
||||
change_key TEXT PRIMARY KEY,
|
||||
change_type TEXT NOT NULL DEFAULT 'match',
|
||||
fiscal_year INTEGER,
|
||||
voucher_no TEXT NOT NULL DEFAULT '',
|
||||
draft_no TEXT NOT NULL DEFAULT '',
|
||||
ledger_date TEXT NOT NULL DEFAULT '',
|
||||
proof_date TEXT NOT NULL DEFAULT '',
|
||||
ledger_account_name TEXT NOT NULL DEFAULT '',
|
||||
voucher_account_name TEXT NOT NULL DEFAULT '',
|
||||
ledger_debit REAL NOT NULL DEFAULT 0,
|
||||
ledger_credit REAL NOT NULL DEFAULT 0,
|
||||
voucher_debit REAL NOT NULL DEFAULT 0,
|
||||
voucher_credit REAL NOT NULL DEFAULT 0,
|
||||
ledger_desc TEXT NOT NULL DEFAULT '',
|
||||
voucher_desc TEXT NOT NULL DEFAULT '',
|
||||
changed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _load_manual_change_keys(conn: sqlite3.Connection) -> tuple[set[str], set[str], set[str]]:
|
||||
_ensure_change_table(conn)
|
||||
review_keys = {
|
||||
str(row[0])
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT review_key
|
||||
FROM wehago_recheck_reviews
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR),
|
||||
).fetchall()
|
||||
}
|
||||
match_change_keys = {
|
||||
str(row[0])
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT change_key
|
||||
FROM wehago_recheck_row_changes
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
AND change_type = 'match'
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR),
|
||||
).fetchall()
|
||||
}
|
||||
split_change_keys = {
|
||||
str(row[0])
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT change_key
|
||||
FROM wehago_recheck_row_changes
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
AND change_type = 'split'
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR),
|
||||
).fetchall()
|
||||
}
|
||||
return review_keys, match_change_keys, split_change_keys
|
||||
|
||||
|
||||
def _snapshot_signature(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT snapshot_signature
|
||||
FROM wehago_snapshot_status
|
||||
WHERE fiscal_year = ?
|
||||
AND state = 'ready'
|
||||
LIMIT 1
|
||||
""",
|
||||
(TARGET_START_YEAR,),
|
||||
).fetchone()
|
||||
if row is not None and str(row["snapshot_signature"] or ""):
|
||||
signature = str(row["snapshot_signature"])
|
||||
exists = conn.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM wehago_compare_export_row_cache
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
AND snapshot_signature = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, signature),
|
||||
).fetchone()
|
||||
if exists:
|
||||
return signature
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT snapshot_signature, COUNT(*) AS row_count, MAX(rowid) AS max_rowid
|
||||
FROM wehago_compare_export_row_cache
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
GROUP BY snapshot_signature
|
||||
ORDER BY
|
||||
CASE WHEN snapshot_signature LIKE 'voucher-summary-v7|recheck-v20%' THEN 0 ELSE 1 END ASC,
|
||||
row_count DESC,
|
||||
max_rowid DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR),
|
||||
).fetchone()
|
||||
if row is None or not str(row["snapshot_signature"] or ""):
|
||||
raise RuntimeError("No snapshot/export signature was found.")
|
||||
return str(row["snapshot_signature"])
|
||||
|
||||
|
||||
def _latest_query_projection_signature(conn: sqlite3.Connection) -> str | None:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT signature, COUNT(DISTINCT status_key) AS status_count, MAX(updated_at) AS updated_at
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ?
|
||||
AND end_year = ?
|
||||
AND signature LIKE ?
|
||||
AND signature NOT LIKE '%|snapshot-recheck-promote|%'
|
||||
GROUP BY signature
|
||||
HAVING status_count >= 5
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, f"{QUERY_PROJECTION_VERSION}|%"),
|
||||
).fetchone()
|
||||
return str(row["signature"] or "") if row is not None else None
|
||||
|
||||
|
||||
def _load_groups_from_query_projection(conn: sqlite3.Connection, signature: str) -> dict[str, list[dict[str, Any]]]:
|
||||
group_rows = conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ?
|
||||
AND end_year = ?
|
||||
AND signature = ?
|
||||
ORDER BY status_key, group_index
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, signature),
|
||||
).fetchall()
|
||||
detail_rows = conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM wehago_compare_query_rows
|
||||
WHERE start_year = ?
|
||||
AND end_year = ?
|
||||
AND signature = ?
|
||||
ORDER BY status_key, group_index, row_index
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, signature),
|
||||
).fetchall()
|
||||
rows_by_key: dict[tuple[str, int], list[dict[str, Any]]] = {}
|
||||
for row in detail_rows:
|
||||
payload = _dict(row)
|
||||
rows_by_key.setdefault((str(payload["status_key"]), int(payload["group_index"])), []).append(payload)
|
||||
groups: dict[str, list[dict[str, Any]]] = {}
|
||||
for row in group_rows:
|
||||
summary = _dict(row)
|
||||
status_key = str(summary["status_key"])
|
||||
group_index = int(summary["group_index"])
|
||||
groups.setdefault(status_key, []).append(
|
||||
{
|
||||
"summary": summary,
|
||||
"rows": rows_by_key.get((status_key, group_index), []),
|
||||
"source_group_index": group_index,
|
||||
}
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def _source_signature(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT signature, COUNT(DISTINCT status_key) AS status_count, COUNT(*) AS group_count
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
AND signature NOT LIKE ?
|
||||
GROUP BY signature
|
||||
HAVING status_count >= 5
|
||||
ORDER BY group_count DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, f"{QUERY_PROJECTION_VERSION}|%"),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise RuntimeError("No source query projection with voucher status groups was found.")
|
||||
return str(row["signature"])
|
||||
|
||||
|
||||
def _matched_source_signature(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT signature, COUNT(*) AS group_count
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
AND status_key = 'voucher_matched'
|
||||
AND signature NOT LIKE ?
|
||||
GROUP BY signature
|
||||
ORDER BY group_count DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, f"{QUERY_PROJECTION_VERSION}|%"),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise RuntimeError("No matched source query projection was found.")
|
||||
return str(row["signature"])
|
||||
|
||||
|
||||
def _load_groups(
|
||||
conn: sqlite3.Connection,
|
||||
signature: str,
|
||||
statuses: tuple[str, ...] | None = None,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
status_filter = ""
|
||||
params: list[Any] = [signature, TARGET_START_YEAR, TARGET_END_YEAR]
|
||||
if statuses:
|
||||
status_filter = f" AND status_key IN ({', '.join('?' for _ in statuses)})"
|
||||
params.extend(statuses)
|
||||
groups: dict[str, list[dict[str, Any]]] = {}
|
||||
group_rows = conn.execute(
|
||||
f"""
|
||||
SELECT *
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE signature = ?
|
||||
AND fiscal_year BETWEEN ? AND ?
|
||||
{status_filter}
|
||||
ORDER BY status_key, group_index
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
detail_rows = conn.execute(
|
||||
f"""
|
||||
SELECT *
|
||||
FROM wehago_compare_query_rows
|
||||
WHERE signature = ?
|
||||
AND fiscal_year BETWEEN ? AND ?
|
||||
{status_filter}
|
||||
ORDER BY status_key, group_index, row_index
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
rows_by_key: dict[tuple[str, int], list[dict[str, Any]]] = {}
|
||||
for row in detail_rows:
|
||||
item = _dict(row)
|
||||
rows_by_key.setdefault((str(item["status_key"]), int(item["group_index"])), []).append(item)
|
||||
for row in group_rows:
|
||||
summary = _dict(row)
|
||||
status_key = str(summary["status_key"])
|
||||
group_index = int(summary["group_index"])
|
||||
groups.setdefault(status_key, []).append(
|
||||
{
|
||||
"summary": summary,
|
||||
"rows": rows_by_key.get((status_key, group_index), []),
|
||||
"source_group_index": group_index,
|
||||
}
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def _load_groups_from_export_cache(conn: sqlite3.Connection, snapshot_signature: str) -> dict[str, list[dict[str, Any]]]:
|
||||
export_rows = conn.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM wehago_compare_export_row_cache
|
||||
WHERE fiscal_year BETWEEN ? AND ?
|
||||
AND snapshot_signature = ?
|
||||
ORDER BY status_key, group_sort, row_sort
|
||||
""",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, snapshot_signature),
|
||||
).fetchall()
|
||||
grouped_rows: dict[tuple[str, int], list[dict[str, Any]]] = {}
|
||||
for row in export_rows:
|
||||
item = _dict(row)
|
||||
grouped_rows.setdefault((str(item["status_key"]), int(item["group_sort"])), []).append(item)
|
||||
|
||||
groups: dict[str, list[dict[str, Any]]] = {}
|
||||
for (status_key, group_index), rows in grouped_rows.items():
|
||||
first = rows[0]
|
||||
detail_rows: list[dict[str, Any]] = []
|
||||
ledger_accounts: list[str] = []
|
||||
voucher_accounts: list[str] = []
|
||||
ledger_vendors: list[str] = []
|
||||
voucher_vendors: list[str] = []
|
||||
|
||||
def append_unique(target: list[str], value: Any) -> None:
|
||||
text_value = clean(value)
|
||||
if text_value and text_value not in target:
|
||||
target.append(text_value)
|
||||
|
||||
ledger_row_count = 0
|
||||
voucher_row_count = 0
|
||||
for row_index, row in enumerate(rows):
|
||||
detail = {
|
||||
"fiscal_year": int(row.get("fiscal_year") or 0),
|
||||
"status_label": "Matched" if status_key in {"voucher_matched", "erp_voucher_matched"} else "Recheck" if status_key == "voucher_recheck" else "Unmatched",
|
||||
"ledger_date": clean(row.get("ledger_date")),
|
||||
"proof_date": "",
|
||||
"voucher_no": clean(row.get("voucher_no")),
|
||||
"draft_no": clean(row.get("draft_no")),
|
||||
"ledger_account_name": clean(row.get("ledger_account_name")),
|
||||
"voucher_account_name": clean(row.get("voucher_account_name")),
|
||||
"ledger_vendor": clean(row.get("ledger_vendor")),
|
||||
"voucher_vendor": clean(row.get("voucher_vendor")),
|
||||
"ledger_debit": float(row.get("ledger_debit") or 0),
|
||||
"ledger_credit": float(row.get("ledger_credit") or 0),
|
||||
"voucher_debit": float(row.get("voucher_debit") or 0),
|
||||
"voucher_credit": float(row.get("voucher_credit") or 0),
|
||||
"ledger_desc": clean(row.get("ledger_desc")),
|
||||
"voucher_desc": clean(row.get("voucher_desc")),
|
||||
"review_reason": "SNAPSHOT_EXPORT_CACHE",
|
||||
"matched_case": "",
|
||||
"ledger_row_key": "",
|
||||
"voucher_row_key": "",
|
||||
"match_identity_key": "",
|
||||
"row_index": row_index,
|
||||
}
|
||||
if detail["ledger_account_name"] or detail["ledger_desc"]:
|
||||
ledger_row_count += 1
|
||||
if detail["voucher_account_name"] or detail["voucher_desc"]:
|
||||
voucher_row_count += 1
|
||||
append_unique(ledger_accounts, detail["ledger_account_name"])
|
||||
append_unique(voucher_accounts, detail["voucher_account_name"])
|
||||
append_unique(ledger_vendors, detail["ledger_vendor"])
|
||||
append_unique(voucher_vendors, detail["voucher_vendor"])
|
||||
detail_rows.append(detail)
|
||||
|
||||
summary = {
|
||||
"fiscal_year": int(first.get("fiscal_year") or 0),
|
||||
"status_label": detail_rows[0]["status_label"] if detail_rows else "",
|
||||
"ledger_date": clean(first.get("ledger_date")),
|
||||
"proof_date": "",
|
||||
"voucher_no": clean(first.get("group_voucher_no")) or clean(first.get("voucher_no")),
|
||||
"draft_no": clean(first.get("group_draft_no")) or clean(first.get("draft_no")),
|
||||
"ledger_row_count": ledger_row_count,
|
||||
"voucher_row_count": voucher_row_count,
|
||||
"ledger_debit": float(first.get("group_ledger_debit") or 0),
|
||||
"ledger_credit": float(first.get("group_ledger_credit") or 0),
|
||||
"voucher_debit": float(first.get("group_voucher_debit") or 0),
|
||||
"voucher_credit": float(first.get("group_voucher_credit") or 0),
|
||||
"ledger_accounts": clean(first.get("group_ledger_accounts")) or ", ".join(ledger_accounts),
|
||||
"voucher_accounts": clean(first.get("group_voucher_accounts")) or ", ".join(voucher_accounts),
|
||||
"ledger_vendors": clean(first.get("group_ledger_vendors")) or ", ".join(ledger_vendors),
|
||||
"voucher_vendors": clean(first.get("group_voucher_vendors")) or ", ".join(voucher_vendors),
|
||||
"review_reason": "SNAPSHOT_EXPORT_CACHE",
|
||||
"search_text": "",
|
||||
}
|
||||
summary["search_text"] = _search_text(summary, detail_rows)
|
||||
groups.setdefault(status_key, []).append(
|
||||
{
|
||||
"summary": summary,
|
||||
"rows": detail_rows,
|
||||
"source_group_index": group_index,
|
||||
}
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def _seed_used_identities(groups: dict[str, list[dict[str, Any]]]) -> tuple[dict[str, int], dict[str, int]]:
|
||||
used_wehago: dict[str, int] = {}
|
||||
used_erp: dict[str, int] = {}
|
||||
for group in groups.get("voucher_matched", []):
|
||||
identity = _wehago_section_identity(group)
|
||||
if identity:
|
||||
used_wehago[identity] = used_wehago.get(identity, 0) + 1
|
||||
for group in groups.get("erp_voucher_matched", []):
|
||||
identity = _erp_section_identity(group)
|
||||
if identity:
|
||||
used_erp[identity] = used_erp.get(identity, 0) + 1
|
||||
return used_wehago, used_erp
|
||||
|
||||
|
||||
def _group_has_manual_match(
|
||||
group: dict[str, Any],
|
||||
review_keys: set[str],
|
||||
match_change_keys: set[str],
|
||||
) -> bool:
|
||||
if _group_manual_review_key(group) in review_keys:
|
||||
return True
|
||||
for row in group.get("rows") or []:
|
||||
if _row_change_key(row, "match") in match_change_keys:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _select_promotions(
|
||||
groups: dict[str, list[dict[str, Any]]],
|
||||
review_keys: set[str],
|
||||
match_change_keys: set[str],
|
||||
) -> set[int]:
|
||||
used_wehago, _used_erp = _seed_used_identities(groups)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
for group in groups.get("voucher_recheck", []):
|
||||
manual_match = _group_has_manual_match(group, review_keys, match_change_keys)
|
||||
if not manual_match and not _is_obvious_recheck_group(group):
|
||||
continue
|
||||
wehago_identity = _wehago_section_identity(group)
|
||||
erp_identity = _erp_section_identity(group)
|
||||
if not wehago_identity or not erp_identity:
|
||||
continue
|
||||
candidates.append(
|
||||
{
|
||||
"group": group,
|
||||
"wehago_identity": wehago_identity,
|
||||
"erp_identity": erp_identity,
|
||||
"wehago_capacity": _section_vat_exception_capacity(group, side="wehago"),
|
||||
"rank": (1 if manual_match else 0, *_recheck_group_rank_key(group)),
|
||||
"manual_match": manual_match,
|
||||
}
|
||||
)
|
||||
candidates.sort(key=lambda item: item["rank"], reverse=True)
|
||||
promoted: set[int] = set()
|
||||
for candidate in candidates:
|
||||
wehago_identity = candidate["wehago_identity"]
|
||||
erp_identity = candidate["erp_identity"]
|
||||
wehago_capacity = int(candidate["wehago_capacity"] or 1)
|
||||
if used_wehago.get(wehago_identity, 0) >= wehago_capacity:
|
||||
continue
|
||||
promoted.add(id(candidate["group"]))
|
||||
used_wehago[wehago_identity] = used_wehago.get(wehago_identity, 0) + 1
|
||||
return promoted
|
||||
|
||||
|
||||
def _search_text(summary: dict[str, Any], rows: list[dict[str, Any]]) -> str:
|
||||
parts = [
|
||||
summary.get("voucher_no"),
|
||||
summary.get("draft_no"),
|
||||
summary.get("ledger_accounts"),
|
||||
summary.get("voucher_accounts"),
|
||||
summary.get("ledger_vendors"),
|
||||
summary.get("voucher_vendors"),
|
||||
summary.get("review_reason"),
|
||||
]
|
||||
for row in rows[:20]:
|
||||
parts.extend([row.get("ledger_desc"), row.get("voucher_desc")])
|
||||
return " ".join(clean(part) for part in parts if clean(part))
|
||||
|
||||
|
||||
def _insert_group(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
signature: str,
|
||||
status_key: str,
|
||||
group_index: int,
|
||||
group: dict[str, Any],
|
||||
promoted: bool = False,
|
||||
split_change_keys: set[str] | None = None,
|
||||
omit_split_rows: bool = False,
|
||||
) -> None:
|
||||
summary = dict(group["summary"])
|
||||
rows = [dict(row) for row in group.get("rows") or []]
|
||||
split_change_keys = split_change_keys or set()
|
||||
if promoted and omit_split_rows:
|
||||
rows = [row for row in rows if _row_change_key(row, "split") not in split_change_keys]
|
||||
summary.update(
|
||||
{
|
||||
"start_year": TARGET_START_YEAR,
|
||||
"end_year": TARGET_END_YEAR,
|
||||
"status_key": status_key,
|
||||
"signature": signature,
|
||||
"group_index": group_index,
|
||||
}
|
||||
)
|
||||
if promoted:
|
||||
summary["review_reason"] = clean(summary.get("review_reason")) or "RECHECK_PROMOTED_BY_USER_RULE"
|
||||
summary["search_text"] = _search_text(summary, rows)
|
||||
values = [summary.get(column, "") for column in GROUP_COLUMNS]
|
||||
conn.execute(
|
||||
f"""
|
||||
INSERT INTO wehago_compare_query_groups ({', '.join(GROUP_COLUMNS)}, created_at, updated_at)
|
||||
VALUES ({', '.join('?' for _ in GROUP_COLUMNS)}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
values,
|
||||
)
|
||||
for row_index, row in enumerate(rows):
|
||||
split_row = promoted and _row_change_key(row, "split") in split_change_keys
|
||||
row.update(
|
||||
{
|
||||
"start_year": TARGET_START_YEAR,
|
||||
"end_year": TARGET_END_YEAR,
|
||||
"status_key": status_key,
|
||||
"signature": signature,
|
||||
"group_index": group_index,
|
||||
"row_index": row_index,
|
||||
}
|
||||
)
|
||||
if promoted:
|
||||
if split_row:
|
||||
row["status_label"] = "Unmatched"
|
||||
row["review_reason"] = "MANUAL_SPLIT_FROM_RECHECK"
|
||||
row["proof_date"] = ""
|
||||
row["draft_no"] = ""
|
||||
row["voucher_account_name"] = ""
|
||||
row["voucher_vendor"] = ""
|
||||
row["voucher_debit"] = 0
|
||||
row["voucher_credit"] = 0
|
||||
row["voucher_desc"] = ""
|
||||
row["voucher_row_key"] = ""
|
||||
row["match_identity_key"] = ""
|
||||
else:
|
||||
row["status_label"] = "Matched"
|
||||
row["review_reason"] = clean(row.get("review_reason")) or "RECHECK_PROMOTED_BY_USER_RULE"
|
||||
values = [row.get(column, "") for column in ROW_COLUMNS]
|
||||
conn.execute(
|
||||
f"""
|
||||
INSERT INTO wehago_compare_query_rows ({', '.join(ROW_COLUMNS)}, created_at, updated_at)
|
||||
VALUES ({', '.join('?' for _ in ROW_COLUMNS)}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
values,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
source_signature = _latest_query_projection_signature(conn)
|
||||
source_kind = "query_projection"
|
||||
if not source_signature:
|
||||
source_signature = _snapshot_signature(conn)
|
||||
source_kind = "export_cache"
|
||||
target_signature = f"{QUERY_PROJECTION_VERSION}|snapshot-recheck-promote|{source_signature}"
|
||||
groups = (
|
||||
_load_groups_from_query_projection(conn, source_signature)
|
||||
if source_kind == "query_projection"
|
||||
else _load_groups_from_export_cache(conn, source_signature)
|
||||
)
|
||||
groups["voucher_recheck"] = [
|
||||
_merge_internal_recheck_pairs(group)
|
||||
for group in groups.get("voucher_recheck", [])
|
||||
]
|
||||
review_keys, match_change_keys, split_change_keys = _load_manual_change_keys(conn)
|
||||
promoted_ids = _select_promotions(groups, review_keys, match_change_keys)
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
conn.execute(
|
||||
"DELETE FROM wehago_compare_query_rows WHERE start_year = ? AND end_year = ? AND signature = ?",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, target_signature),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM wehago_compare_query_groups WHERE start_year = ? AND end_year = ? AND signature = ?",
|
||||
(TARGET_START_YEAR, TARGET_END_YEAR, target_signature),
|
||||
)
|
||||
max_group_index = {
|
||||
status_key: max([int(group["source_group_index"]) for group in status_groups] or [0])
|
||||
for status_key, status_groups in groups.items()
|
||||
}
|
||||
for status_key, status_groups in groups.items():
|
||||
for group in status_groups:
|
||||
if status_key == "voucher_recheck" and id(group) in promoted_ids:
|
||||
continue
|
||||
_insert_group(
|
||||
conn,
|
||||
signature=target_signature,
|
||||
status_key=status_key,
|
||||
group_index=int(group["source_group_index"]),
|
||||
group=group,
|
||||
)
|
||||
for group in groups.get("voucher_recheck", []):
|
||||
if id(group) not in promoted_ids:
|
||||
continue
|
||||
max_group_index["voucher_matched"] = max_group_index.get("voucher_matched", 0) + 1
|
||||
_insert_group(
|
||||
conn,
|
||||
signature=target_signature,
|
||||
status_key="voucher_matched",
|
||||
group_index=max_group_index["voucher_matched"],
|
||||
group=group,
|
||||
promoted=True,
|
||||
split_change_keys=split_change_keys,
|
||||
)
|
||||
max_group_index["erp_voucher_matched"] = max_group_index.get("erp_voucher_matched", 0) + 1
|
||||
_insert_group(
|
||||
conn,
|
||||
signature=target_signature,
|
||||
status_key="erp_voucher_matched",
|
||||
group_index=max_group_index["erp_voucher_matched"],
|
||||
group=group,
|
||||
promoted=True,
|
||||
split_change_keys=split_change_keys,
|
||||
omit_split_rows=True,
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO wehago_action_history (action_type, payload_json, created_at)
|
||||
VALUES ('auto_recheck_promote', ?, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(
|
||||
f'{{"count": {len(promoted_ids)}, "start_year": {TARGET_START_YEAR}, '
|
||||
f'"manual_review_keys": {len(review_keys)}, "match_change_keys": {len(match_change_keys)}, '
|
||||
f'"split_change_keys": {len(split_change_keys)}, '
|
||||
f'"source_kind": "{source_kind}", '
|
||||
f'"end_year": {TARGET_END_YEAR}, "signature": "{target_signature}", '
|
||||
f'"created_at": "{datetime.now().isoformat(timespec="seconds")}"}}',
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
counts = conn.execute(
|
||||
"""
|
||||
SELECT status_key, COUNT(*)
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE signature = ?
|
||||
AND fiscal_year BETWEEN ? AND ?
|
||||
GROUP BY status_key
|
||||
ORDER BY status_key
|
||||
""",
|
||||
(target_signature, TARGET_START_YEAR, TARGET_END_YEAR),
|
||||
).fetchall()
|
||||
print(
|
||||
{
|
||||
"source_signature": source_signature,
|
||||
"source_kind": source_kind,
|
||||
"target_signature": target_signature,
|
||||
"promoted": len(promoted_ids),
|
||||
"counts": {str(row[0]): int(row[1]) for row in counts},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
from pathlib import Path
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from main import engine
|
||||
from wehago_compare import (
|
||||
_build_db_state_signature,
|
||||
_get_fast_year_export_row_cache_signature,
|
||||
_load_snapshot_status_map,
|
||||
_project_year_export_row_cache_from_latest_resolved,
|
||||
_rebuild_compare_query_projection,
|
||||
_refresh_year_resolved_sections,
|
||||
_status_row_counts_from_sections,
|
||||
_upsert_snapshot_status,
|
||||
)
|
||||
|
||||
|
||||
def parse_range(value: str) -> tuple[int, int]:
|
||||
raw = str(value or "").strip()
|
||||
if "-" not in raw:
|
||||
year = int(raw)
|
||||
return year, year
|
||||
left, right = raw.split("-", 1)
|
||||
start_year = int(left)
|
||||
end_year = int(right)
|
||||
if start_year > end_year:
|
||||
start_year, end_year = end_year, start_year
|
||||
return start_year, end_year
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Rebuild WEHAGO compare snapshots and query projections.")
|
||||
parser.add_argument("--years", nargs="*", type=int, default=[])
|
||||
parser.add_argument("--fast-years", nargs="*", type=int, default=[])
|
||||
parser.add_argument("--ranges", nargs="*", default=[])
|
||||
args = parser.parse_args()
|
||||
|
||||
lock_path = Path("/tmp/wehago_compare_compute.lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with lock_path.open("w") as lock_file:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||
for year in sorted({int(year) for year in args.fast_years if int(year or 0) > 0}):
|
||||
signature = _build_db_state_signature(conn, year, year)
|
||||
print({"step": "fast_year_projection_start", "year": year, "signature": signature}, flush=True)
|
||||
projected = _project_year_export_row_cache_from_latest_resolved(conn, year, signature)
|
||||
if not projected:
|
||||
print({"step": "fast_year_projection_miss", "year": year}, flush=True)
|
||||
_refresh_year_resolved_sections(conn, year)
|
||||
else:
|
||||
_upsert_snapshot_status(
|
||||
conn,
|
||||
year,
|
||||
signature=signature,
|
||||
state="ready",
|
||||
row_counts={},
|
||||
built_now=True,
|
||||
)
|
||||
selected_signature = _get_fast_year_export_row_cache_signature(conn, year)
|
||||
print(
|
||||
{
|
||||
"step": "fast_year_projection_done",
|
||||
"year": year,
|
||||
"selected_signature": selected_signature,
|
||||
},
|
||||
flush=True,
|
||||
)
|
||||
for year in sorted({int(year) for year in args.years if int(year or 0) > 0}):
|
||||
signature = _build_db_state_signature(conn, year, year)
|
||||
print({"step": "year_snapshot_start", "year": year, "signature": signature}, flush=True)
|
||||
_refresh_year_resolved_sections(conn, year)
|
||||
print({"step": "year_snapshot_done", "year": year}, flush=True)
|
||||
for start_year, end_year in [parse_range(item) for item in args.ranges]:
|
||||
print({"step": "projection_start", "start_year": start_year, "end_year": end_year}, flush=True)
|
||||
counts, snapshot_state = _rebuild_compare_query_projection(engine, conn, start_year, end_year)
|
||||
print(
|
||||
{
|
||||
"step": "projection_done",
|
||||
"start_year": start_year,
|
||||
"end_year": end_year,
|
||||
"counts": counts,
|
||||
"snapshot_state": snapshot_state,
|
||||
},
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -35,7 +35,7 @@ if ss -ltn "( sport = :${PORT} )" | tail -n +2 | grep -q .; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export INTRANET_AUTO_RELOAD="${INTRANET_AUTO_RELOAD:-1}"
|
||||
export INTRANET_AUTO_RELOAD="${INTRANET_AUTO_RELOAD:-0}"
|
||||
export INTRANET_PORT="$PORT"
|
||||
echo "서버를 시작합니다: http://127.0.0.1:${PORT} (auto_reload=${INTRANET_AUTO_RELOAD})"
|
||||
echo "예기치 않게 종료되면 2초 후 자동 재시작합니다."
|
||||
|
||||
@@ -81,6 +81,31 @@ function Get-WslListeningPorts {
|
||||
return $portMap.Values | Sort-Object Port
|
||||
}
|
||||
|
||||
function Get-DockerPublishedPorts {
|
||||
$publishedPorts = New-Object System.Collections.Generic.HashSet[int]
|
||||
try {
|
||||
$raw = docker ps --format "{{.Ports}}" 2>$null
|
||||
if (-not $raw) {
|
||||
return $publishedPorts
|
||||
}
|
||||
|
||||
foreach ($line in ($raw -split "`r?`n")) {
|
||||
if ([string]::IsNullOrWhiteSpace($line)) {
|
||||
continue
|
||||
}
|
||||
foreach ($match in [regex]::Matches($line, "(?:0\.0\.0\.0|\[::\]):(?<port>\d+)->")) {
|
||||
$port = 0
|
||||
if ([int]::TryParse($match.Groups["port"].Value, [ref]$port)) {
|
||||
[void]$publishedPorts.Add($port)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return $publishedPorts
|
||||
}
|
||||
return $publishedPorts
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($WslIp)) {
|
||||
$detected = wsl.exe hostname -I 2>$null
|
||||
if (-not $detected) {
|
||||
@@ -110,6 +135,10 @@ $listenAddresses = $listenAddresses | Where-Object { -not [string]::IsNullOrWhit
|
||||
Write-Host "Setting Windows portproxy for all active WSL TCP ports..." -ForegroundColor Cyan
|
||||
Write-Host "WSL IP: $WslIp"
|
||||
$portEntries = @(Get-WslListeningPorts)
|
||||
$dockerPublishedPorts = @(Get-DockerPublishedPorts)
|
||||
if ($dockerPublishedPorts.Count -gt 0) {
|
||||
Write-Host "Docker-published Windows ports: $((@($dockerPublishedPorts) | Sort-Object) -join ', ')" -ForegroundColor Cyan
|
||||
}
|
||||
|
||||
$knownPortEntries = @()
|
||||
foreach ($knownPort in ($knownPorts.Keys | Sort-Object)) {
|
||||
@@ -162,7 +191,13 @@ foreach ($entry in $reachableEntries) {
|
||||
|
||||
foreach ($address in $listenAddresses) {
|
||||
netsh interface portproxy delete v4tov4 listenport=$port listenaddress=$address | Out-Null
|
||||
netsh interface portproxy add v4tov4 listenport=$port listenaddress=$address connectport=$port connectaddress=$WslIp
|
||||
if ($dockerPublishedPorts -notcontains $port) {
|
||||
netsh interface portproxy add v4tov4 listenport=$port listenaddress=$address connectport=$port connectaddress=$WslIp
|
||||
}
|
||||
}
|
||||
|
||||
if ($dockerPublishedPorts -contains $port) {
|
||||
Write-Host " Docker already publishes this port on Windows; stale portproxy entries were removed and only firewall is refreshed." -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
$existingRule = Get-NetFirewallRule -DisplayName $ruleName -ErrorAction SilentlyContinue
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from runtime_config import ( # noqa: E402
|
||||
BACKUP_DIR,
|
||||
DB_PATH,
|
||||
WAL_BLOCK_HEAVY_BYTES,
|
||||
WAL_WARN_BYTES,
|
||||
ensure_runtime_directories,
|
||||
sqlite_runtime_status,
|
||||
)
|
||||
|
||||
|
||||
CACHE_TABLES = (
|
||||
"wehago_compare_export_row_cache",
|
||||
"wehago_compare_query_page_cache",
|
||||
"wehago_compare_query_groups",
|
||||
"wehago_compare_query_rows",
|
||||
"wehago_metric_count_cache",
|
||||
"wehago_pair_recommend_cache",
|
||||
"wehago_raw_erp_trace_candidate_cache",
|
||||
"wehago_result_row_cache",
|
||||
"wehago_summary_range_cache",
|
||||
)
|
||||
SOURCE_TABLES = ("wehago_voucher_rows", "wehago_ledger_rows")
|
||||
|
||||
|
||||
def _size(path: Path) -> int:
|
||||
return path.stat().st_size if path.exists() else 0
|
||||
|
||||
|
||||
def _table_counts(conn: sqlite3.Connection, tables: tuple[str, ...]) -> dict[str, int | None]:
|
||||
existing = {
|
||||
str(row[0])
|
||||
for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
|
||||
}
|
||||
counts: dict[str, int | None] = {}
|
||||
for table in tables:
|
||||
if table not in existing:
|
||||
counts[table] = None
|
||||
continue
|
||||
counts[table] = int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
|
||||
return counts
|
||||
|
||||
|
||||
def _table_storage(conn: sqlite3.Connection, tables: tuple[str, ...]) -> dict[str, dict[str, int | None]]:
|
||||
existing = {
|
||||
str(row[0])
|
||||
for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
|
||||
}
|
||||
has_dbstat = True
|
||||
try:
|
||||
conn.execute("SELECT 1 FROM dbstat LIMIT 1").fetchone()
|
||||
except sqlite3.DatabaseError:
|
||||
has_dbstat = False
|
||||
page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
|
||||
storage: dict[str, dict[str, int | None]] = {}
|
||||
for table in tables:
|
||||
if table not in existing:
|
||||
storage[table] = {"rows": None, "bytes": None}
|
||||
continue
|
||||
rows = int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
|
||||
bytes_used: int | None = None
|
||||
if has_dbstat:
|
||||
try:
|
||||
stat = conn.execute(
|
||||
"SELECT SUM(pgsize) FROM dbstat WHERE name = ?",
|
||||
(table,),
|
||||
).fetchone()
|
||||
bytes_used = int(stat[0] or 0)
|
||||
except sqlite3.DatabaseError:
|
||||
bytes_used = None
|
||||
storage[table] = {"rows": rows, "bytes": bytes_used if has_dbstat else None}
|
||||
if not has_dbstat:
|
||||
for table in storage.values():
|
||||
table["estimated_page_size"] = page_size
|
||||
return storage
|
||||
|
||||
|
||||
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
||||
return bool(
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
|
||||
(table,),
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def cache_retention_report(limit: int) -> dict[str, Any]:
|
||||
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
|
||||
try:
|
||||
if not _table_exists(conn, "wehago_compare_export_row_cache"):
|
||||
return {"export_row_cache": {"exists": False}}
|
||||
signatures = [
|
||||
{
|
||||
"fiscal_year": int(row[0]),
|
||||
"snapshot_signature": str(row[1]),
|
||||
"rows": int(row[2]),
|
||||
}
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT fiscal_year, snapshot_signature, COUNT(*) AS rows
|
||||
FROM wehago_compare_export_row_cache
|
||||
GROUP BY fiscal_year, snapshot_signature
|
||||
ORDER BY fiscal_year DESC, rows DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
]
|
||||
orphan_rows = 0
|
||||
if _table_exists(conn, "wehago_snapshot_status"):
|
||||
orphan_rows = int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_export_row_cache c
|
||||
LEFT JOIN wehago_snapshot_status s
|
||||
ON s.fiscal_year = c.fiscal_year
|
||||
AND s.snapshot_signature = c.snapshot_signature
|
||||
AND s.state = 'ready'
|
||||
WHERE s.fiscal_year IS NULL
|
||||
"""
|
||||
).fetchone()[0]
|
||||
)
|
||||
return {
|
||||
"export_row_cache": {
|
||||
"exists": True,
|
||||
"signature_groups_sample": signatures,
|
||||
"orphan_rows_not_matching_ready_snapshot": orphan_rows,
|
||||
}
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def prune_orphan_export_cache(dry_run: bool, acknowledged: bool) -> dict[str, Any]:
|
||||
if not dry_run and not acknowledged:
|
||||
raise RuntimeError(
|
||||
"실제 삭제는 `--ack-delete-rebuildable-cache`를 함께 지정해야 합니다. "
|
||||
"삭제 대상은 현재 ready 스냅샷 서명과 맞지 않는 재생성 가능 export-row 캐시입니다."
|
||||
)
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
try:
|
||||
if not _table_exists(conn, "wehago_compare_export_row_cache") or not _table_exists(
|
||||
conn,
|
||||
"wehago_snapshot_status",
|
||||
):
|
||||
return {"dry_run": dry_run, "candidate_rows": 0, "deleted_rows": 0}
|
||||
candidate_rows = int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_export_row_cache c
|
||||
LEFT JOIN wehago_snapshot_status s
|
||||
ON s.fiscal_year = c.fiscal_year
|
||||
AND s.snapshot_signature = c.snapshot_signature
|
||||
AND s.state = 'ready'
|
||||
WHERE s.fiscal_year IS NULL
|
||||
"""
|
||||
).fetchone()[0]
|
||||
)
|
||||
deleted_rows = 0
|
||||
if not dry_run and candidate_rows:
|
||||
conn.execute(
|
||||
"""
|
||||
DELETE FROM wehago_compare_export_row_cache
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM wehago_snapshot_status s
|
||||
WHERE s.fiscal_year = wehago_compare_export_row_cache.fiscal_year
|
||||
AND s.snapshot_signature = wehago_compare_export_row_cache.snapshot_signature
|
||||
AND s.state = 'ready'
|
||||
)
|
||||
"""
|
||||
)
|
||||
deleted_rows = conn.total_changes
|
||||
conn.commit()
|
||||
return {"dry_run": dry_run, "candidate_rows": candidate_rows, "deleted_rows": deleted_rows}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def query_projection_retention_report(keep: int) -> dict[str, Any]:
|
||||
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
if not _table_exists(conn, "wehago_compare_query_groups"):
|
||||
return {"query_projection_cache": {"exists": False}}
|
||||
groups = conn.execute(
|
||||
"""
|
||||
SELECT start_year, end_year, signature,
|
||||
COUNT(*) AS group_rows,
|
||||
MAX(updated_at) AS max_updated_at
|
||||
FROM wehago_compare_query_groups
|
||||
GROUP BY start_year, end_year, signature
|
||||
ORDER BY start_year DESC, end_year DESC, max_updated_at DESC
|
||||
"""
|
||||
).fetchall()
|
||||
keep = max(1, int(keep))
|
||||
by_scope: dict[tuple[int, int], list[sqlite3.Row]] = {}
|
||||
for row in groups:
|
||||
by_scope.setdefault((int(row["start_year"]), int(row["end_year"])), []).append(row)
|
||||
obsolete: list[sqlite3.Row] = []
|
||||
retained: list[dict[str, Any]] = []
|
||||
for scope, rows in by_scope.items():
|
||||
for idx, row in enumerate(rows):
|
||||
item = {
|
||||
"start_year": int(row["start_year"]),
|
||||
"end_year": int(row["end_year"]),
|
||||
"signature": str(row["signature"] or ""),
|
||||
"group_rows": int(row["group_rows"] or 0),
|
||||
"max_updated_at": str(row["max_updated_at"] or ""),
|
||||
"retained": idx < keep,
|
||||
}
|
||||
if idx < keep:
|
||||
retained.append(item)
|
||||
else:
|
||||
obsolete.append(row)
|
||||
obsolete_group_rows = sum(int(row["group_rows"] or 0) for row in obsolete)
|
||||
obsolete_query_rows = 0
|
||||
obsolete_page_rows = 0
|
||||
for row in obsolete:
|
||||
params = (
|
||||
int(row["start_year"]),
|
||||
int(row["end_year"]),
|
||||
str(row["signature"] or ""),
|
||||
)
|
||||
if _table_exists(conn, "wehago_compare_query_rows"):
|
||||
obsolete_query_rows += int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_query_rows
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
if _table_exists(conn, "wehago_compare_query_page_cache"):
|
||||
obsolete_page_rows += int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_query_page_cache
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
return {
|
||||
"query_projection_cache": {
|
||||
"exists": True,
|
||||
"keep_per_range": keep,
|
||||
"range_signature_count": len(groups),
|
||||
"obsolete_signature_count": len(obsolete),
|
||||
"obsolete_group_rows": obsolete_group_rows,
|
||||
"obsolete_query_rows": obsolete_query_rows,
|
||||
"obsolete_page_rows": obsolete_page_rows,
|
||||
"retained_sample": retained[:20],
|
||||
}
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def prune_old_query_projections(dry_run: bool, acknowledged: bool, keep: int) -> dict[str, Any]:
|
||||
if not dry_run and not acknowledged:
|
||||
raise RuntimeError(
|
||||
"실제 삭제는 `--ack-delete-rebuildable-cache`를 함께 지정해야 합니다. "
|
||||
"삭제 대상은 범위별 최신 N개를 제외한 재생성 가능 query projection 캐시입니다."
|
||||
)
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
if not _table_exists(conn, "wehago_compare_query_groups"):
|
||||
return {"dry_run": dry_run, "candidate_group_rows": 0, "candidate_query_rows": 0, "candidate_page_rows": 0}
|
||||
keep = max(1, int(keep))
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT start_year, end_year, signature, MAX(updated_at) AS max_updated_at
|
||||
FROM wehago_compare_query_groups
|
||||
GROUP BY start_year, end_year, signature
|
||||
ORDER BY start_year, end_year, max_updated_at DESC
|
||||
"""
|
||||
).fetchall()
|
||||
by_scope: dict[tuple[int, int], list[sqlite3.Row]] = {}
|
||||
for row in rows:
|
||||
by_scope.setdefault((int(row["start_year"]), int(row["end_year"])), []).append(row)
|
||||
obsolete = [row for scope_rows in by_scope.values() for row in scope_rows[keep:]]
|
||||
candidate_group_rows = 0
|
||||
candidate_query_rows = 0
|
||||
candidate_page_rows = 0
|
||||
deleted_rows = 0
|
||||
for row in obsolete:
|
||||
params = (
|
||||
int(row["start_year"]),
|
||||
int(row["end_year"]),
|
||||
str(row["signature"] or ""),
|
||||
)
|
||||
candidate_group_rows += int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_query_groups
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
if _table_exists(conn, "wehago_compare_query_rows"):
|
||||
candidate_query_rows += int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_query_rows
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
if _table_exists(conn, "wehago_compare_query_page_cache"):
|
||||
candidate_page_rows += int(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM wehago_compare_query_page_cache
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
""",
|
||||
params,
|
||||
).fetchone()[0]
|
||||
)
|
||||
if not dry_run:
|
||||
for table in (
|
||||
"wehago_compare_query_page_cache",
|
||||
"wehago_compare_query_rows",
|
||||
"wehago_compare_query_groups",
|
||||
):
|
||||
if not _table_exists(conn, table):
|
||||
continue
|
||||
before = conn.total_changes
|
||||
conn.execute(
|
||||
f"""
|
||||
DELETE FROM {table}
|
||||
WHERE start_year = ? AND end_year = ? AND signature = ?
|
||||
""",
|
||||
params,
|
||||
)
|
||||
deleted_rows += conn.total_changes - before
|
||||
conn.commit()
|
||||
return {
|
||||
"dry_run": dry_run,
|
||||
"keep_per_range": keep,
|
||||
"obsolete_signature_count": len(obsolete),
|
||||
"candidate_group_rows": candidate_group_rows,
|
||||
"candidate_query_rows": candidate_query_rows,
|
||||
"candidate_page_rows": candidate_page_rows,
|
||||
"deleted_rows": deleted_rows,
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def status(include_counts: bool) -> dict[str, Any]:
|
||||
path = DB_PATH
|
||||
payload: dict[str, Any] = {
|
||||
"database_path": str(path),
|
||||
"database_exists": path.exists(),
|
||||
"files": {
|
||||
"database_bytes": _size(path),
|
||||
"wal_bytes": _size(path.with_name(path.name + "-wal")),
|
||||
"shm_bytes": _size(path.with_name(path.name + "-shm")),
|
||||
},
|
||||
"runtime": sqlite_runtime_status(),
|
||||
"thresholds": {
|
||||
"wal_warn_bytes": WAL_WARN_BYTES,
|
||||
"wal_block_heavy_bytes": WAL_BLOCK_HEAVY_BYTES,
|
||||
},
|
||||
}
|
||||
if not path.exists():
|
||||
return payload
|
||||
conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5)
|
||||
try:
|
||||
payload["database"] = {
|
||||
"journal_mode": str(conn.execute("PRAGMA journal_mode").fetchone()[0]),
|
||||
"page_size": int(conn.execute("PRAGMA page_size").fetchone()[0]),
|
||||
"page_count": int(conn.execute("PRAGMA page_count").fetchone()[0]),
|
||||
"freelist_count": int(conn.execute("PRAGMA freelist_count").fetchone()[0]),
|
||||
"wal_autocheckpoint": int(conn.execute("PRAGMA wal_autocheckpoint").fetchone()[0]),
|
||||
}
|
||||
if include_counts:
|
||||
payload["source_table_counts"] = _table_counts(conn, SOURCE_TABLES)
|
||||
payload["cache_table_counts"] = _table_counts(conn, CACHE_TABLES)
|
||||
payload["cache_table_storage"] = _table_storage(conn, CACHE_TABLES)
|
||||
finally:
|
||||
conn.close()
|
||||
wal_bytes = int(payload["files"]["wal_bytes"])
|
||||
warnings: list[str] = []
|
||||
if not payload["runtime"]["meets_minimum_safe_version"]:
|
||||
warnings.append(
|
||||
"SQLite 런타임이 3.51.3 미만입니다. WAL DB를 운영하기 전에 안전 런타임으로 교체하세요."
|
||||
)
|
||||
if wal_bytes >= WAL_BLOCK_HEAVY_BYTES:
|
||||
warnings.append("WAL이 중단 기준을 넘었습니다. 신규 대량 캐시 재생성을 보류하고 유지보수 창을 확보하세요.")
|
||||
elif wal_bytes >= WAL_WARN_BYTES:
|
||||
warnings.append("WAL이 경고 기준을 넘었습니다. 긴 조회/쓰기 작업과 checkpoint 상태를 점검하세요.")
|
||||
payload["warnings"] = warnings
|
||||
return payload
|
||||
|
||||
|
||||
def backup(output: Path | None, verify: str) -> dict[str, Any]:
|
||||
if not DB_PATH.exists():
|
||||
raise FileNotFoundError(DB_PATH)
|
||||
ensure_runtime_directories()
|
||||
output = output or BACKUP_DIR / f"data-migration-{datetime.now():%Y%m%d-%H%M%S}.sqlite3"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = output.with_name("." + output.name + ".tmp")
|
||||
source_conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
|
||||
target_conn = sqlite3.connect(temp_path)
|
||||
last_reported_percent = -10
|
||||
|
||||
def report_progress(_status: int, remaining: int, total: int) -> None:
|
||||
nonlocal last_reported_percent
|
||||
percent = int((total - remaining) * 100 / total) if total else 100
|
||||
reported_percent = min(100, (percent // 10) * 10)
|
||||
if reported_percent > last_reported_percent:
|
||||
print(f"backup copy progress: {reported_percent}%", file=sys.stderr, flush=True)
|
||||
last_reported_percent = reported_percent
|
||||
|
||||
try:
|
||||
source_conn.backup(target_conn, pages=8192, progress=report_progress)
|
||||
check = "skipped"
|
||||
if verify == "smoke":
|
||||
target_conn.execute("PRAGMA schema_version").fetchone()
|
||||
target_conn.execute("SELECT COUNT(*) FROM sqlite_master").fetchone()
|
||||
check = "open-and-schema-readable"
|
||||
elif verify != "none":
|
||||
pragma = "integrity_check" if verify == "full" else "quick_check"
|
||||
print(f"backup verification started: {pragma}", file=sys.stderr, flush=True)
|
||||
check = str(target_conn.execute(f"PRAGMA {pragma}").fetchone()[0])
|
||||
if check.lower() != "ok":
|
||||
raise RuntimeError(f"백업 {pragma} 실패: {check}")
|
||||
finally:
|
||||
target_conn.close()
|
||||
source_conn.close()
|
||||
temp_path.replace(output)
|
||||
return {"backup_path": str(output), "backup_bytes": _size(output), "verification": verify, "check_result": check}
|
||||
|
||||
|
||||
def checkpoint(mode: str, acknowledged: bool) -> dict[str, Any]:
|
||||
if not acknowledged:
|
||||
raise RuntimeError(
|
||||
"checkpoint는 운영 서버와 대량 작업을 중단한 유지보수 창에서만 실행하세요. "
|
||||
"`--ack-maintenance-window`를 함께 지정해야 합니다."
|
||||
)
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
try:
|
||||
before = _size(DB_PATH.with_name(DB_PATH.name + "-wal"))
|
||||
result = conn.execute(f"PRAGMA wal_checkpoint({mode.upper()})").fetchone()
|
||||
after = _size(DB_PATH.with_name(DB_PATH.name + "-wal"))
|
||||
finally:
|
||||
conn.close()
|
||||
return {"mode": mode, "checkpoint_result": list(result or ()), "wal_bytes_before": before, "wal_bytes_after": after}
|
||||
|
||||
|
||||
def prepare_runtime_layout(root: Path) -> dict[str, str]:
|
||||
paths = {
|
||||
"db": root / "db",
|
||||
"cache": root / "cache",
|
||||
"backups": root / "backups",
|
||||
"exports": root / "exports",
|
||||
}
|
||||
for path in paths.values():
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
env_file = root / "runtime.env.example"
|
||||
env_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
f"INTRANET_DB_PATH={paths['db'] / 'data.db'}",
|
||||
f"INTRANET_BACKUP_DIR={paths['backups']}",
|
||||
f"INTRANET_CACHE_ROOT={paths['cache']}",
|
||||
f"INTRANET_COMPARE_EXPORT_DIR={paths['exports'] / 'wehago_compare'}",
|
||||
"INTRANET_REQUIRE_SAFE_SQLITE=1",
|
||||
f"WEHAGO_SOURCE_ROOT={Path.home() / 'WEHAGO_DB'}",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return {key: str(path) for key, path in paths.items()} | {"env_example": str(env_file)}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="SQLite/WAL runtime inspection and maintenance helpers.")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
status_parser = subparsers.add_parser("status")
|
||||
status_parser.add_argument("--include-counts", action="store_true")
|
||||
|
||||
backup_parser = subparsers.add_parser("backup")
|
||||
backup_parser.add_argument("--output", type=Path)
|
||||
backup_parser.add_argument(
|
||||
"--verify",
|
||||
choices=("smoke", "quick", "full", "none"),
|
||||
default="smoke",
|
||||
help="smoke is suitable for container trials; use quick/full during a maintenance window.",
|
||||
)
|
||||
|
||||
checkpoint_parser = subparsers.add_parser("checkpoint")
|
||||
checkpoint_parser.add_argument("--mode", choices=("passive", "full", "restart", "truncate"), default="passive")
|
||||
checkpoint_parser.add_argument("--ack-maintenance-window", action="store_true")
|
||||
|
||||
layout_parser = subparsers.add_parser("prepare-layout")
|
||||
layout_parser.add_argument("--root", type=Path, default=Path.home() / "intranet-runtime")
|
||||
|
||||
report_parser = subparsers.add_parser("cache-retention-report")
|
||||
report_parser.add_argument("--limit", type=int, default=20)
|
||||
|
||||
prune_parser = subparsers.add_parser("prune-orphan-export-cache")
|
||||
prune_parser.add_argument("--execute", action="store_true")
|
||||
prune_parser.add_argument("--ack-delete-rebuildable-cache", action="store_true")
|
||||
|
||||
query_report_parser = subparsers.add_parser("query-retention-report")
|
||||
query_report_parser.add_argument("--keep", type=int, default=2)
|
||||
|
||||
query_prune_parser = subparsers.add_parser("prune-old-query-projections")
|
||||
query_prune_parser.add_argument("--keep", type=int, default=2)
|
||||
query_prune_parser.add_argument("--execute", action="store_true")
|
||||
query_prune_parser.add_argument("--ack-delete-rebuildable-cache", action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.command == "status":
|
||||
result = status(args.include_counts)
|
||||
elif args.command == "backup":
|
||||
result = backup(args.output, args.verify)
|
||||
elif args.command == "checkpoint":
|
||||
result = checkpoint(args.mode, args.ack_maintenance_window)
|
||||
elif args.command == "cache-retention-report":
|
||||
result = cache_retention_report(args.limit)
|
||||
elif args.command == "prune-orphan-export-cache":
|
||||
result = prune_orphan_export_cache(
|
||||
dry_run=not args.execute,
|
||||
acknowledged=args.ack_delete_rebuildable_cache,
|
||||
)
|
||||
elif args.command == "query-retention-report":
|
||||
result = query_projection_retention_report(args.keep)
|
||||
elif args.command == "prune-old-query-projections":
|
||||
result = prune_old_query_projections(
|
||||
dry_run=not args.execute,
|
||||
acknowledged=args.ack_delete_rebuildable_cache,
|
||||
keep=args.keep,
|
||||
)
|
||||
else:
|
||||
result = prepare_runtime_layout(args.root)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user