From 54def57e99082f6fb354e0c22784597cee5d62f8 Mon Sep 17 00:00:00 2001 From: Hyein Date: Tue, 9 Jun 2026 10:12:15 +0900 Subject: [PATCH] Improve 8092 cache loading and SQLite stability --- mysql_preview_server.py | 181 ++++++++++++++++++++++++++++------ project-codes.html | 38 +++---- project_code_viewer_server.py | 2 +- 3 files changed, 161 insertions(+), 60 deletions(-) diff --git a/mysql_preview_server.py b/mysql_preview_server.py index e262ffd..20b0e9b 100644 --- a/mysql_preview_server.py +++ b/mysql_preview_server.py @@ -48,10 +48,26 @@ HOLIDAY_DATES = { SITE_SYNC_JOBS = {} SITE_SYNC_LOCK = threading.Lock() +DB_INIT_LOCK = threading.Lock() +DB_SCHEMA_READY = False + + +def configure_sqlite_connection(conn): + conn.execute('PRAGMA journal_mode=WAL') + conn.execute('PRAGMA synchronous=NORMAL') + conn.execute('PRAGMA busy_timeout=30000') + return conn + + +def open_db_connection(timeout=30): + conn = sqlite3.connect(DB_PATH, timeout=timeout) + return configure_sqlite_connection(conn) def init_db(conn): + global DB_SCHEMA_READY + configure_sqlite_connection(conn) conn.executescript( ''' CREATE TABLE IF NOT EXISTS member ( @@ -151,6 +167,13 @@ def init_db(conn): PRIMARY KEY (projectCode, workDate, selMenu) ); + CREATE TABLE IF NOT EXISTS member_site_identity ( + memberNo TEXT PRIMARY KEY, + korName TEXT DEFAULT '', + juminno TEXT DEFAULT '', + updatedAt TEXT DEFAULT '' + ); + CREATE TABLE IF NOT EXISTS work_calendar_day ( memberNo TEXT, workDate TEXT, @@ -289,17 +312,25 @@ def init_db(conn): conn.execute("CREATE INDEX IF NOT EXISTS idx_erp_bridge_overview_cache_page ON erp_bridge_overview_cache(sourcePage, projectCode, bridgeNo)") conn.execute("CREATE INDEX IF NOT EXISTS idx_erp_budget_plan_cache_page ON erp_budget_plan_cache(sourcePage, projectCode)") conn.commit() + DB_SCHEMA_READY = True def ensure_db_schema(conn): - try: - init_db(conn) - cleanup_site_records_after_retire(conn) - except sqlite3.OperationalError as error: - if 'locked' in str(error).lower(): - return False - raise - return True + global DB_SCHEMA_READY + if DB_SCHEMA_READY: + return True + with DB_INIT_LOCK: + if DB_SCHEMA_READY: + return True + try: + configure_sqlite_connection(conn) + init_db(conn) + cleanup_site_records_after_retire(conn) + except sqlite3.OperationalError as error: + if 'locked' in str(error).lower(): + return False + raise + return True def normalize_member_no(v): @@ -351,8 +382,8 @@ def _member_rows_for_site_work_day(name_to_members, kor_name, work_date): def cleanup_site_records_after_retire(conn): - # 이미 저장된 사업관리 기록 중 같은 이름의 현재 사번이 존재하는데 - # 퇴사 사번의 퇴사일 이후로 붙은 기록은 중복 집계 원인이므로 제거한다. + # 사업관리 원본은 이름 중심이라 퇴사자와 동명이인/재입사자를 잘못 매칭할 수 있다. + # member 퇴사일 이후의 사업관리 기록은 해당 퇴사 사번에서 무조건 제거한다. conn.execute( ''' DELETE FROM site_worksheet_record @@ -362,16 +393,6 @@ def cleanup_site_records_after_retire(conn): JOIN member oldm ON oldm.MemberNo = s.memberNo WHERE IFNULL(oldm.retireFlag, '') NOT IN ('', '0000-00-00', '0000-00-00 00:00:00') AND date(s.workDate) > date(substr(oldm.retireFlag, 1, 10)) - AND EXISTS ( - SELECT 1 - FROM member newm - WHERE newm.korName = oldm.korName - AND newm.MemberNo <> oldm.MemberNo - AND ( - IFNULL(newm.retireFlag, '') IN ('', '0000-00-00', '0000-00-00 00:00:00') - OR date(s.workDate) <= date(substr(newm.retireFlag, 1, 10)) - ) - ) ) ''' ) @@ -1583,6 +1604,66 @@ def _load_construct_paymonth_rows(session, member_name, juminno, write_day): return _parse_construct_paymonth_rows(html, write_day) +def _project_code_for_bridge_name(conn, bridge_name, fallback_project_code=''): + bridge_name = _as_text(bridge_name).strip() + if not bridge_name: + return fallback_project_code or '' + row = conn.execute( + ''' + SELECT projectCode + FROM project_alias + WHERE shortName = ? + ORDER BY projectCode DESC + LIMIT 1 + ''', + (bridge_name,) + ).fetchone() + if row: + return row[0] or fallback_project_code or '' + row = conn.execute( + ''' + SELECT projectCode + FROM project_alias + WHERE shortName LIKE ? + ORDER BY LENGTH(shortName) ASC, projectCode DESC + LIMIT 1 + ''', + (f'%{bridge_name}%',) + ).fetchone() + return (row[0] if row else '') or fallback_project_code or '' + + +def insert_construct_paymonth_records(conn, session, member_no, member_name, juminno, write_day, start_date, end_date, fallback_project_code=''): + rows = _load_construct_paymonth_rows(session, member_name, juminno, write_day) + inserted = 0 + for row in rows: + work_date = row.get('workDate') or '' + if work_date < start_date or work_date > end_date: + continue + project_code = _project_code_for_bridge_name(conn, row.get('bridgeName', ''), fallback_project_code) + if not project_code: + continue + conn.execute( + ''' + INSERT OR REPLACE INTO site_worksheet_record + (projectCode, workDate, memberNo, korName, jobType, workText, note, personCount) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', + ( + project_code, + work_date, + member_no, + member_name, + row.get('jobType', '') or '', + row.get('bridgeName', '') or row.get('workText', '') or '', + row.get('workText', '') or '', + _float_or_zero(row.get('personCount')), + ) + ) + inserted += 1 + return inserted + + def _site_record_rows_from_cache(conn, start_date, end_date, member_no): cur = conn.execute( ''' @@ -2078,7 +2159,24 @@ def get_member_site_worksheet_records(conn, start_date, end_date, member_no, ref 'cachedRows': len(cached), }) if not targets: - return {'rows': cached, 'source': 'cache'} + ident = conn.execute( + ''' + SELECT IFNULL(juminno, '') + FROM member_site_identity + WHERE memberNo = ? + ''', + (member_no,) + ).fetchone() + if not ident or not (ident[0] or '').strip(): + return {'rows': cached, 'source': 'cache'} + if progress is not None: + progress['phase'] = '월별 근무현황 보강' + s = erp_login_session() + insert_construct_paymonth_records(conn, s, member_no, member_name, ident[0].strip(), end_date, start_date, end_date) + conn.commit() + cleanup_site_records_after_retire(conn) + rebuild_work_calendar_tables(conn) + return {'rows': _site_record_rows_from_cache(conn, start_date, end_date, member_no), 'source': 'erp_paymonth', 'added': 0} if progress is not None: progress['phase'] = '사업관리 로그인' @@ -2205,6 +2303,25 @@ def get_member_site_worksheet_records(conn, start_date, end_date, member_no, ref 'processedTargets': int(progress.get('processedTargets') or 0) + 1, 'added': len(out), }) + ident = conn.execute( + ''' + SELECT IFNULL(juminno, '') + FROM member_site_identity + WHERE memberNo = ? + ''', + (member_no,) + ).fetchone() + if ident and (ident[0] or '').strip(): + if progress is not None: + progress['phase'] = '월별 근무현황 보강' + try: + inserted = insert_construct_paymonth_records( + conn, s, member_no, member_name, ident[0].strip(), end_date, start_date, end_date + ) + if inserted and progress is not None: + progress['added'] = int(progress.get('added') or 0) + inserted + except Exception: + pass if progress is not None: progress['phase'] = '달력 테이블 정리' conn.commit() @@ -3838,7 +3955,7 @@ class Handler(BaseHTTPRequestHandler): q = parse_qs(parsed.query) page_name = q.get('page', ['const'])[0] refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes') - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) if refresh: search_text = q.get('searchText', [''])[0] @@ -3874,7 +3991,7 @@ class Handler(BaseHTTPRequestHandler): project_code = q.get('projectCode', [''])[0] project_name = q.get('projectName', [''])[0] refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes') - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) if refresh: detail = fetch_erp_contract_detail(page_name, project_code, project_name) @@ -3896,7 +4013,7 @@ class Handler(BaseHTTPRequestHandler): project_code = q.get('projectCode', [''])[0] project_name = q.get('projectName', [''])[0] refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes') - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) if refresh: result = fetch_erp_bridge_overviews(page_name, project_code, project_name) @@ -3917,7 +4034,7 @@ class Handler(BaseHTTPRequestHandler): project_code = q.get('projectCode', [''])[0] project_name = q.get('projectName', [''])[0] refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes') - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) if refresh: result = fetch_erp_budget_plan(page_name, project_code, project_name) @@ -3928,7 +4045,7 @@ class Handler(BaseHTTPRequestHandler): return self._json(200, {'ok': True, 'plan': cached}) if parsed.path == '/api/stats': - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: return self._json(200, get_stats(conn)) if parsed.path == '/api/rebuild-work-calendar': @@ -4105,7 +4222,7 @@ class Handler(BaseHTTPRequestHandler): q = parse_qs(parsed.query) page_name = q.get('page', ['const'])[0] result = fetch_erp_project_codes(page_name) - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) sync_info = replace_erp_project_code_cache(conn, result['page'], result['rows']) return self._json( @@ -4128,7 +4245,7 @@ class Handler(BaseHTTPRequestHandler): project_code = q.get('projectCode', [''])[0] project_name = q.get('projectName', [''])[0] detail = fetch_erp_contract_detail(page_name, project_code, project_name) - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) sync_info = replace_erp_contract_detail_cache(conn, detail) cached = get_erp_contract_detail_cache(conn, page_name, project_code) @@ -4151,7 +4268,7 @@ class Handler(BaseHTTPRequestHandler): project_code = q.get('projectCode', [''])[0] project_name = q.get('projectName', [''])[0] result = fetch_erp_bridge_overviews(page_name, project_code, project_name) - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) sync_info = replace_erp_bridge_overview_cache(conn, result) cached = get_erp_bridge_overview_cache(conn, page_name, project_code) @@ -4175,7 +4292,7 @@ class Handler(BaseHTTPRequestHandler): project_code = q.get('projectCode', [''])[0] project_name = q.get('projectName', [''])[0] result = fetch_erp_budget_plan(page_name, project_code, project_name) - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) sync_info = replace_erp_budget_plan_cache(conn, result) cached = get_erp_budget_plan_cache(conn, page_name, project_code) @@ -4188,7 +4305,7 @@ class Handler(BaseHTTPRequestHandler): return self._json(404, {'error': 'Not found'}) try: - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: ensure_db_schema(conn) used = load_from_mysql_into_sqlite(conn) try: @@ -4233,7 +4350,7 @@ def local_ip(): if __name__ == '__main__': os.makedirs(BASE_DIR, exist_ok=True) - with sqlite3.connect(DB_PATH) as conn: + with open_db_connection() as conn: init_db(conn) try: alias_rows = load_project_alias_from_erp() diff --git a/project-codes.html b/project-codes.html index 21e0f10..1199145 100644 --- a/project-codes.html +++ b/project-codes.html @@ -824,7 +824,7 @@ function renderDetail(detail) { if (!detail) { detailTitle.textContent = '계약정보 표'; - detailMeta.textContent = '시공코드나 약칭을 클릭하면 DB에 저장된 계약정보를 먼저 보여줍니다.'; + detailMeta.textContent = '시공코드나 약칭을 클릭하면 DB 캐시를 먼저 보여줍니다. 최신 정보가 필요하면 새로 가져오기를 누르세요.'; detailBody.innerHTML = '선택된 시공코드가 없습니다.'; bridgeMeta.textContent = '공사규모와 공사개요를 교량명 기준으로 매칭해서 보여줍니다.'; bridgeBody.innerHTML = '선택된 시공코드가 없습니다.'; @@ -1148,10 +1148,10 @@ } detailMeta.textContent = refresh ? 'ERP 계약정보를 DB로 동기화하는 중입니다.' - : 'DB에 저장된 계약정보를 불러오는 중입니다.'; + : 'DB 캐시에 저장된 계약정보를 불러오는 중입니다.'; bridgeMeta.textContent = refresh ? 'ERP 공사규모와 공사개요를 동기화하는 중입니다.' - : 'DB에 저장된 공사규모와 공사개요를 불러오는 중입니다.'; + : 'DB 캐시에 저장된 공사규모와 공사개요를 불러오는 중입니다.'; detailSyncButton.disabled = true; try { const detailUrl = `/api/erp-contract-detail?page=const&projectCode=${encodeURIComponent(projectCode)}&projectName=${encodeURIComponent(projectName || '')}&refresh=${refresh ? '1' : '0'}`; @@ -1168,37 +1168,21 @@ if (!detailResponse.ok) { if (!refresh && detailResponse.status === 404) { - return loadDetail(projectCode, projectName, true); + state.detail = null; + state.bridgeOverviews = overviewResponse.ok ? (Array.isArray(overviewData.overviews) ? overviewData.overviews : []) : []; + state.budgetPlan = budgetPlanResponse.ok ? (budgetPlanData.plan || null) : null; + detailTitle.textContent = `${projectName || '-'} [${projectCode}]`; + detailMeta.textContent = 'DB 캐시에 계약정보가 없습니다. 새 정보 다시 가져오기를 누르면 ERP에서 수집합니다.'; + detailBody.innerHTML = '저장된 계약정보가 없습니다. 최신 정보가 필요하면 오른쪽 상단 버튼을 눌러주세요.'; + renderMergedBridgeRows([], state.bridgeOverviews || [], state.budgetPlan); + return; } throw new Error(data.error || '계약정보 조회에 실패했습니다.'); } state.detail = data.detail || null; - if (!refresh && (!state.detail || !Array.isArray(state.detail.scaleRows) || !state.detail.scaleRows.length)) { - return loadDetail(projectCode, projectName, true); - } state.bridgeOverviews = Array.isArray(overviewData.overviews) ? overviewData.overviews : []; state.budgetPlan = budgetPlanResponse.ok ? (budgetPlanData.plan || null) : null; - if (!refresh && !state.bridgeOverviews.length) { - const freshOverviewResponse = await fetch( - `/api/erp-bridge-overviews?page=const&projectCode=${encodeURIComponent(projectCode)}&projectName=${encodeURIComponent(projectName || '')}&refresh=1`, - { cache: 'no-store' } - ); - const freshOverviewData = await freshOverviewResponse.json(); - if (freshOverviewResponse.ok) { - state.bridgeOverviews = Array.isArray(freshOverviewData.overviews) ? freshOverviewData.overviews : []; - } - } - if (!refresh && !state.budgetPlan) { - const freshPlanResponse = await fetch( - `/api/erp-budget-plan?page=const&projectCode=${encodeURIComponent(projectCode)}&projectName=${encodeURIComponent(projectName || '')}&refresh=1`, - { cache: 'no-store' } - ); - const freshPlanData = await freshPlanResponse.json(); - if (freshPlanResponse.ok) { - state.budgetPlan = freshPlanData.plan || null; - } - } if (state.selectedRow && state.detail) { state.selectedRow.contractType = state.detail.contractType || state.selectedRow.contractType || ''; state.selectedRow.applicationType = Array.from(new Set( diff --git a/project_code_viewer_server.py b/project_code_viewer_server.py index 2d0df6d..4d2f801 100644 --- a/project_code_viewer_server.py +++ b/project_code_viewer_server.py @@ -18,7 +18,7 @@ class ProjectCodeViewerHandler(base.Handler): if __name__ == '__main__': os.makedirs(base.BASE_DIR, exist_ok=True) try: - with sqlite3.connect(base.DB_PATH, timeout=30) as conn: + with base.open_db_connection() as conn: base.init_db(conn) except sqlite3.OperationalError as error: if 'locked' in str(error).lower():