Improve 8092 cache loading and SQLite stability

mysql_preview_server.py

@@ -48,10 +48,26 @@ HOLIDAY_DATES = {
 
 SITE_SYNC_JOBS = {}
 SITE_SYNC_LOCK = threading.Lock()
+DB_INIT_LOCK = threading.Lock()
+DB_SCHEMA_READY = False
+
+
+def configure_sqlite_connection(conn):
+    conn.execute('PRAGMA journal_mode=WAL')
+    conn.execute('PRAGMA synchronous=NORMAL')
+    conn.execute('PRAGMA busy_timeout=30000')
+    return conn
+
+
+def open_db_connection(timeout=30):
+    conn = sqlite3.connect(DB_PATH, timeout=timeout)
+    return configure_sqlite_connection(conn)
 
 
 
 def init_db(conn):
+    global DB_SCHEMA_READY
+    configure_sqlite_connection(conn)
     conn.executescript(
         '''
         CREATE TABLE IF NOT EXISTS member (
@@ -151,6 +167,13 @@ def init_db(conn):
             PRIMARY KEY (projectCode, workDate, selMenu)
         );
 
+        CREATE TABLE IF NOT EXISTS member_site_identity (
+            memberNo TEXT PRIMARY KEY,
+            korName TEXT DEFAULT '',
+            juminno TEXT DEFAULT '',
+            updatedAt TEXT DEFAULT ''
+        );
+
         CREATE TABLE IF NOT EXISTS work_calendar_day (
             memberNo TEXT,
             workDate TEXT,
@@ -289,17 +312,25 @@ def init_db(conn):
     conn.execute("CREATE INDEX IF NOT EXISTS idx_erp_bridge_overview_cache_page ON erp_bridge_overview_cache(sourcePage, projectCode, bridgeNo)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_erp_budget_plan_cache_page ON erp_budget_plan_cache(sourcePage, projectCode)")
     conn.commit()
+    DB_SCHEMA_READY = True
 
 
 def ensure_db_schema(conn):
-    try:
-        init_db(conn)
-        cleanup_site_records_after_retire(conn)
-    except sqlite3.OperationalError as error:
-        if 'locked' in str(error).lower():
-            return False
-        raise
-    return True
+    global DB_SCHEMA_READY
+    if DB_SCHEMA_READY:
+        return True
+    with DB_INIT_LOCK:
+        if DB_SCHEMA_READY:
+            return True
+        try:
+            configure_sqlite_connection(conn)
+            init_db(conn)
+            cleanup_site_records_after_retire(conn)
+        except sqlite3.OperationalError as error:
+            if 'locked' in str(error).lower():
+                return False
+            raise
+        return True
 
 
 def normalize_member_no(v):
@@ -351,8 +382,8 @@ def _member_rows_for_site_work_day(name_to_members, kor_name, work_date):
 
 
 def cleanup_site_records_after_retire(conn):
-    # 이미 저장된 사업관리 기록 중 같은 이름의 현재 사번이 존재하는데
-    # 퇴사 사번의 퇴사일 이후로 붙은 기록은 중복 집계 원인이므로 제거한다.
+    # 사업관리 원본은 이름 중심이라 퇴사자와 동명이인/재입사자를 잘못 매칭할 수 있다.
+    # member 퇴사일 이후의 사업관리 기록은 해당 퇴사 사번에서 무조건 제거한다.
     conn.execute(
         '''
         DELETE FROM site_worksheet_record
@@ -362,16 +393,6 @@ def cleanup_site_records_after_retire(conn):
           JOIN member oldm ON oldm.MemberNo = s.memberNo
           WHERE IFNULL(oldm.retireFlag, '') NOT IN ('', '0000-00-00', '0000-00-00 00:00:00')
             AND date(s.workDate) > date(substr(oldm.retireFlag, 1, 10))
-            AND EXISTS (
-              SELECT 1
-              FROM member newm
-              WHERE newm.korName = oldm.korName
-                AND newm.MemberNo <> oldm.MemberNo
-                AND (
-                  IFNULL(newm.retireFlag, '') IN ('', '0000-00-00', '0000-00-00 00:00:00')
-                  OR date(s.workDate) <= date(substr(newm.retireFlag, 1, 10))
-                )
-            )
         )
         '''
     )
@@ -1583,6 +1604,66 @@ def _load_construct_paymonth_rows(session, member_name, juminno, write_day):
     return _parse_construct_paymonth_rows(html, write_day)
 
 
+def _project_code_for_bridge_name(conn, bridge_name, fallback_project_code=''):
+    bridge_name = _as_text(bridge_name).strip()
+    if not bridge_name:
+        return fallback_project_code or ''
+    row = conn.execute(
+        '''
+        SELECT projectCode
+        FROM project_alias
+        WHERE shortName = ?
+        ORDER BY projectCode DESC
+        LIMIT 1
+        ''',
+        (bridge_name,)
+    ).fetchone()
+    if row:
+        return row[0] or fallback_project_code or ''
+    row = conn.execute(
+        '''
+        SELECT projectCode
+        FROM project_alias
+        WHERE shortName LIKE ?
+        ORDER BY LENGTH(shortName) ASC, projectCode DESC
+        LIMIT 1
+        ''',
+        (f'%{bridge_name}%',)
+    ).fetchone()
+    return (row[0] if row else '') or fallback_project_code or ''
+
+
+def insert_construct_paymonth_records(conn, session, member_no, member_name, juminno, write_day, start_date, end_date, fallback_project_code=''):
+    rows = _load_construct_paymonth_rows(session, member_name, juminno, write_day)
+    inserted = 0
+    for row in rows:
+        work_date = row.get('workDate') or ''
+        if work_date < start_date or work_date > end_date:
+            continue
+        project_code = _project_code_for_bridge_name(conn, row.get('bridgeName', ''), fallback_project_code)
+        if not project_code:
+            continue
+        conn.execute(
+            '''
+            INSERT OR REPLACE INTO site_worksheet_record
+            (projectCode, workDate, memberNo, korName, jobType, workText, note, personCount)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            ''',
+            (
+                project_code,
+                work_date,
+                member_no,
+                member_name,
+                row.get('jobType', '') or '',
+                row.get('bridgeName', '') or row.get('workText', '') or '',
+                row.get('workText', '') or '',
+                _float_or_zero(row.get('personCount')),
+            )
+        )
+        inserted += 1
+    return inserted
+
+
 def _site_record_rows_from_cache(conn, start_date, end_date, member_no):
     cur = conn.execute(
         '''
@@ -2078,7 +2159,24 @@ def get_member_site_worksheet_records(conn, start_date, end_date, member_no, ref
             'cachedRows': len(cached),
         })
     if not targets:
-        return {'rows': cached, 'source': 'cache'}
+        ident = conn.execute(
+            '''
+            SELECT IFNULL(juminno, '')
+            FROM member_site_identity
+            WHERE memberNo = ?
+            ''',
+            (member_no,)
+        ).fetchone()
+        if not ident or not (ident[0] or '').strip():
+            return {'rows': cached, 'source': 'cache'}
+        if progress is not None:
+            progress['phase'] = '월별 근무현황 보강'
+        s = erp_login_session()
+        insert_construct_paymonth_records(conn, s, member_no, member_name, ident[0].strip(), end_date, start_date, end_date)
+        conn.commit()
+        cleanup_site_records_after_retire(conn)
+        rebuild_work_calendar_tables(conn)
+        return {'rows': _site_record_rows_from_cache(conn, start_date, end_date, member_no), 'source': 'erp_paymonth', 'added': 0}
 
     if progress is not None:
         progress['phase'] = '사업관리 로그인'
@@ -2205,6 +2303,25 @@ def get_member_site_worksheet_records(conn, start_date, end_date, member_no, ref
                 'processedTargets': int(progress.get('processedTargets') or 0) + 1,
                 'added': len(out),
             })
+    ident = conn.execute(
+        '''
+        SELECT IFNULL(juminno, '')
+        FROM member_site_identity
+        WHERE memberNo = ?
+        ''',
+        (member_no,)
+    ).fetchone()
+    if ident and (ident[0] or '').strip():
+        if progress is not None:
+            progress['phase'] = '월별 근무현황 보강'
+        try:
+            inserted = insert_construct_paymonth_records(
+                conn, s, member_no, member_name, ident[0].strip(), end_date, start_date, end_date
+            )
+            if inserted and progress is not None:
+                progress['added'] = int(progress.get('added') or 0) + inserted
+        except Exception:
+            pass
     if progress is not None:
         progress['phase'] = '달력 테이블 정리'
     conn.commit()
@@ -3838,7 +3955,7 @@ class Handler(BaseHTTPRequestHandler):
                 q = parse_qs(parsed.query)
                 page_name = q.get('page', ['const'])[0]
                 refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes')
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     if refresh:
                         search_text = q.get('searchText', [''])[0]
@@ -3874,7 +3991,7 @@ class Handler(BaseHTTPRequestHandler):
                 project_code = q.get('projectCode', [''])[0]
                 project_name = q.get('projectName', [''])[0]
                 refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes')
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     if refresh:
                         detail = fetch_erp_contract_detail(page_name, project_code, project_name)
@@ -3896,7 +4013,7 @@ class Handler(BaseHTTPRequestHandler):
                 project_code = q.get('projectCode', [''])[0]
                 project_name = q.get('projectName', [''])[0]
                 refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes')
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     if refresh:
                         result = fetch_erp_bridge_overviews(page_name, project_code, project_name)
@@ -3917,7 +4034,7 @@ class Handler(BaseHTTPRequestHandler):
                 project_code = q.get('projectCode', [''])[0]
                 project_name = q.get('projectName', [''])[0]
                 refresh = q.get('refresh', ['0'])[0] in ('1', 'true', 'yes')
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     if refresh:
                         result = fetch_erp_budget_plan(page_name, project_code, project_name)
@@ -3928,7 +4045,7 @@ class Handler(BaseHTTPRequestHandler):
                 return self._json(200, {'ok': True, 'plan': cached})
 
             if parsed.path == '/api/stats':
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     return self._json(200, get_stats(conn))
 
             if parsed.path == '/api/rebuild-work-calendar':
@@ -4105,7 +4222,7 @@ class Handler(BaseHTTPRequestHandler):
                 q = parse_qs(parsed.query)
                 page_name = q.get('page', ['const'])[0]
                 result = fetch_erp_project_codes(page_name)
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     sync_info = replace_erp_project_code_cache(conn, result['page'], result['rows'])
                 return self._json(
@@ -4128,7 +4245,7 @@ class Handler(BaseHTTPRequestHandler):
                 project_code = q.get('projectCode', [''])[0]
                 project_name = q.get('projectName', [''])[0]
                 detail = fetch_erp_contract_detail(page_name, project_code, project_name)
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     sync_info = replace_erp_contract_detail_cache(conn, detail)
                     cached = get_erp_contract_detail_cache(conn, page_name, project_code)
@@ -4151,7 +4268,7 @@ class Handler(BaseHTTPRequestHandler):
                 project_code = q.get('projectCode', [''])[0]
                 project_name = q.get('projectName', [''])[0]
                 result = fetch_erp_bridge_overviews(page_name, project_code, project_name)
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     sync_info = replace_erp_bridge_overview_cache(conn, result)
                     cached = get_erp_bridge_overview_cache(conn, page_name, project_code)
@@ -4175,7 +4292,7 @@ class Handler(BaseHTTPRequestHandler):
                 project_code = q.get('projectCode', [''])[0]
                 project_name = q.get('projectName', [''])[0]
                 result = fetch_erp_budget_plan(page_name, project_code, project_name)
-                with sqlite3.connect(DB_PATH) as conn:
+                with open_db_connection() as conn:
                     ensure_db_schema(conn)
                     sync_info = replace_erp_budget_plan_cache(conn, result)
                     cached = get_erp_budget_plan_cache(conn, page_name, project_code)
@@ -4188,7 +4305,7 @@ class Handler(BaseHTTPRequestHandler):
             return self._json(404, {'error': 'Not found'})
 
         try:
-            with sqlite3.connect(DB_PATH) as conn:
+            with open_db_connection() as conn:
                 ensure_db_schema(conn)
                 used = load_from_mysql_into_sqlite(conn)
                 try:
@@ -4233,7 +4350,7 @@ def local_ip():
 
 if __name__ == '__main__':
     os.makedirs(BASE_DIR, exist_ok=True)
-    with sqlite3.connect(DB_PATH) as conn:
+    with open_db_connection() as conn:
         init_db(conn)
         try:
             alias_rows = load_project_alias_from_erp()