크론구조 개선

This commit is contained in:
Lectom C Han
2025-12-10 09:55:35 +09:00
parent 8762db2c0e
commit b1a9204e22
11 changed files with 426 additions and 102 deletions

View File

@@ -7,7 +7,7 @@
- Keep `cmd/server` thin; place new logic in `internal/<domain>` with clear boundaries.
## Build, Test, and Development Commands
- `PORT=8080 GEOIP_DB_PATH=./GeoLite2-City.mmdb go run ./cmd/server` runs the API locally without Docker.
- `SERVICE_PORT=8080 GEOIP_DB_PATH=./GeoLite2-City.mmdb go run ./cmd/server` runs the API locally without Docker.
- `docker compose up --build` builds and starts the containerized service (mounts the local database).
- `curl "http://localhost:8080/lookup?ip=1.1.1.1"` exercises the lookup endpoint; omit `ip` to use the callers address.
- `go build ./...` validates compilation before pushing changes.

View File

@@ -10,14 +10,12 @@ RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -o /bin/geoip ./cmd/server && \
CGO_ENABLED=0 go build -o /bin/geoip-loader ./cmd/loader && \
CGO_ENABLED=0 go build -o /bin/user-program-import ./cmd/user_program_import
CGO_ENABLED=0 go build -o /bin/user-program-import ./cmd/user_program_import && \
CGO_ENABLED=0 go build -o /bin/user-program-dump ./cmd/user_program_dump && \
CGO_ENABLED=0 go build -o /bin/user-program-sync ./cmd/user_program_sync
FROM debian:trixie-slim
RUN apt-get update && \
apt-get install -y --no-install-recommends mysql-client && \
rm -rf /var/lib/apt/lists/*
RUN useradd --create-home --shell /usr/sbin/nologin appuser
WORKDIR /app
@@ -25,11 +23,10 @@ WORKDIR /app
COPY --from=builder /bin/geoip /usr/local/bin/geoip
COPY --from=builder /bin/geoip-loader /usr/local/bin/geoip-loader
COPY --from=builder /bin/user-program-import /usr/local/bin/user-program-import
COPY --from=builder /bin/user-program-dump /usr/local/bin/user-program-dump
COPY --from=builder /bin/user-program-sync /usr/local/bin/user-program-sync
COPY initial_data /app/initial_data
COPY scripts /app/scripts
RUN mkdir -p /app/update_data /app/log && \
chmod 0755 /app/scripts/dump_and_import.sh && \
chmod -R 0755 /app/scripts
RUN mkdir -p /app/update_data
ENV GEOIP_DB_PATH=/app/initial_data/GeoLite2-City.mmdb
USER appuser

View File

@@ -6,6 +6,7 @@ import (
"log"
"net/url"
"os"
"strings"
"time"
"github.com/gofiber/fiber/v2"
@@ -17,8 +18,8 @@ import (
const (
defaultPort = "8080"
defaultDBPath = "/initial_data/GeoLite2-City.mmdb"
defaultCron = ""
defaultScript = "./scripts/dump_and_import.sh"
defaultCron = "5 0 * * *" // 매일 00:05 KST
defaultJob = "user-program-sync"
)
func main() {
@@ -26,7 +27,7 @@ func main() {
dbPath := env("GEOIP_DB_PATH", defaultDBPath)
dbURL := os.Getenv("DATABASE_URL")
lookupQuery := os.Getenv("GEOIP_LOOKUP_QUERY")
port := env("PORT", defaultPort)
port := env("SERVICE_PORT", defaultPort)
resolver, err := geo.NewResolver(geo.Config{
Backend: backend,
@@ -112,6 +113,21 @@ func env(key, fallback string) string {
return fallback
}
func envBool(key string, fallback bool) bool {
val := os.Getenv(key)
if val == "" {
return fallback
}
switch strings.ToLower(val) {
case "1", "t", "true", "y", "yes", "on":
return true
case "0", "f", "false", "n", "no", "off":
return false
default:
return fallback
}
}
func sanitizeDBURL(raw string) string {
u, err := url.Parse(raw)
if err != nil {
@@ -121,15 +137,16 @@ func sanitizeDBURL(raw string) string {
}
func maybeStartScheduler() func() context.Context {
cronExpr := env("USER_PROGRAM_CRON", defaultCron)
if cronExpr == "" {
enabled := envBool("USER_PROGRAM_CRON_ENABLE", false)
if !enabled {
return nil
}
script := env("USER_PROGRAM_SCRIPT", defaultScript)
cronExpr := defaultCron
command := defaultJob
sched, err := schedule.Start(schedule.Config{
CronExpr: cronExpr,
ScriptPath: script,
Command: command,
Logger: log.Default(),
})
if err != nil {

View File

@@ -0,0 +1,295 @@
package main
import (
"context"
"database/sql"
"encoding/csv"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
"time"
"github.com/go-sql-driver/mysql"
)
const (
defaultUpdateDir = "./update_data"
defaultTable = "user_program_info"
defaultDB = "user_program_info"
defaultDumpTimeout = 5 * time.Minute
)
type config struct {
host string
port int
user string
password string
database string
table string
updateDir string
target time.Time
}
func main() {
logger := log.New(os.Stdout, "[dump] ", log.LstdFlags)
cfg, err := loadConfig()
if err != nil {
log.Fatalf("config error: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), defaultDumpTimeout)
defer cancel()
dsn := (&mysql.Config{
User: cfg.user,
Passwd: cfg.password,
Net: "tcp",
Addr: netAddr(cfg.host, cfg.port),
DBName: cfg.database,
Params: map[string]string{"parseTime": "true", "loc": "UTC", "charset": "utf8mb4"},
AllowNativePasswords: true,
}).FormatDSN()
db, err := sql.Open("mysql", dsn)
if err != nil {
log.Fatalf("failed to open mysql connection: %v", err)
}
defer db.Close()
if err := db.PingContext(ctx); err != nil {
log.Fatalf("failed to ping mysql: %v", err)
}
if _, err := db.ExecContext(ctx, "SET time_zone = '+00:00'"); err != nil {
log.Fatalf("failed to set timezone: %v", err)
}
outPath, err := dumpToCSV(ctx, db, cfg)
if err != nil {
log.Fatalf("dump failed: %v", err)
}
logger.Printf("dumped %s to %s", cfg.target.Format("2006-01-02"), outPath)
}
func loadConfig() (config, error) {
port, err := strconv.Atoi(env("USER_PROGRAM_INFO_PORT", "3306"))
if err != nil {
return config{}, fmt.Errorf("invalid USER_PROGRAM_INFO_PORT: %w", err)
}
target, err := resolveTargetDate(env("USER_PROGRAM_TARGET_DATE", ""))
if err != nil {
return config{}, err
}
table := env("USER_PROGRAM_INFO_TABLE", defaultTable)
if !regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(table) {
return config{}, fmt.Errorf("invalid table name: %s", table)
}
updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir)
if updateDir == "" {
updateDir = defaultUpdateDir
}
if err := os.MkdirAll(updateDir, 0o755); err != nil {
return config{}, fmt.Errorf("creating update dir: %w", err)
}
return config{
host: envRequired("USER_PROGRAM_INFO_HOST"),
port: port,
user: envRequired("USER_PROGRAM_INFO_USERNAME"),
password: envRequired("USER_PROGRAM_INFO_PASSWORD"),
database: env("USER_PROGRAM_INFO_DB", defaultDB),
table: table,
updateDir: updateDir,
target: target,
}, nil
}
func dumpToCSV(ctx context.Context, db *sql.DB, cfg config) (string, error) {
query := fmt.Sprintf(`
SELECT
id,
product_name,
login_id,
user_employee_id,
login_version,
login_public_ip,
login_local_ip,
user_company,
user_department,
user_position,
user_login_time,
created_at,
user_family_flag
FROM %s
WHERE DATE(CONVERT_TZ(created_at, '+00:00', '+09:00')) = ?;`, cfg.table)
rows, err := db.QueryContext(ctx, query, cfg.target.Format("2006-01-02"))
if err != nil {
return "", err
}
defer rows.Close()
filename := fmt.Sprintf("user_program_info_%s.csv", cfg.target.Format("20060102"))
outPath := filepath.Join(cfg.updateDir, filename)
tmpPath := outPath + ".tmp"
f, err := os.Create(tmpPath)
if err != nil {
return "", err
}
defer f.Close()
writer := csv.NewWriter(f)
defer writer.Flush()
header := []string{
"id",
"product_name",
"login_id",
"user_employee_id",
"login_version",
"login_public_ip",
"login_local_ip",
"user_company",
"user_department",
"user_position",
"user_login_time",
"created_at",
"user_family_flag",
}
if err := writer.Write(header); err != nil {
return "", err
}
for rows.Next() {
record, err := scanRow(rows)
if err != nil {
return "", err
}
if err := writer.Write(record); err != nil {
return "", err
}
}
if err := rows.Err(); err != nil {
return "", err
}
writer.Flush()
if err := writer.Error(); err != nil {
return "", err
}
if err := os.Rename(tmpPath, outPath); err != nil {
return "", err
}
return outPath, nil
}
func scanRow(rows *sql.Rows) ([]string, error) {
var (
id sql.NullInt64
productName sql.NullString
loginID sql.NullString
employeeID sql.NullString
loginVersion sql.NullString
loginPublicIP sql.NullString
loginLocalIP sql.NullString
userCompany sql.NullString
userDepartment sql.NullString
userPosition sql.NullString
userLoginTime sql.NullString
createdAt sql.NullString
userFamilyFlag sql.NullString
)
if err := rows.Scan(
&id,
&productName,
&loginID,
&employeeID,
&loginVersion,
&loginPublicIP,
&loginLocalIP,
&userCompany,
&userDepartment,
&userPosition,
&userLoginTime,
&createdAt,
&userFamilyFlag,
); err != nil {
return nil, err
}
if !id.Valid {
return nil, fmt.Errorf("row missing id")
}
return []string{
strconv.FormatInt(id.Int64, 10),
nullToString(productName),
nullToString(loginID),
nullToString(employeeID),
nullToString(loginVersion),
nullToString(loginPublicIP),
nullToString(loginLocalIP),
nullToString(userCompany),
nullToString(userDepartment),
nullToString(userPosition),
nullToString(userLoginTime),
nullToString(createdAt),
nullToString(userFamilyFlag),
}, nil
}
func resolveTargetDate(raw string) (time.Time, error) {
if raw == "" {
now := time.Now().In(kst())
yesterday := now.AddDate(0, 0, -1)
return time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, kst()), nil
}
t, err := time.ParseInLocation("2006-01-02", raw, kst())
if err != nil {
return time.Time{}, fmt.Errorf("invalid USER_PROGRAM_TARGET_DATE: %w", err)
}
return t, nil
}
func kst() *time.Location {
loc, err := time.LoadLocation("Asia/Seoul")
if err != nil {
return time.FixedZone("KST", 9*60*60)
}
return loc
}
func env(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func envRequired(key string) string {
v := os.Getenv(key)
if v == "" {
log.Fatalf("%s is required", key)
}
return v
}
func netAddr(host string, port int) string {
return fmt.Sprintf("%s:%d", host, port)
}
func nullToString(v sql.NullString) string {
if v.Valid {
return v.String
}
return ""
}

View File

@@ -0,0 +1,75 @@
package main
import (
"context"
"log"
"os"
"os/exec"
"path/filepath"
"time"
)
const (
defaultUpdateDir = "/app/update_data"
defaultLogDir = "/app/log"
defaultSchema = "public"
defaultTimeout = 15 * time.Minute
)
func main() {
logger := log.New(os.Stdout, "[sync] ", log.LstdFlags)
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
defer cancel()
updateDir := env("USER_PROGRAM_UPDATE_DIR", defaultUpdateDir)
logDir := env("USER_PROGRAM_IMPORT_LOG_DIR", defaultLogDir)
schema := env("USER_PROGRAM_INFO_SCHEMA", defaultSchema)
ensureDir(updateDir, logger)
ensureDir(logDir, logger)
if err := runCmd(ctx, logger, "user-program-dump", map[string]string{
"USER_PROGRAM_UPDATE_DIR": updateDir,
}); err != nil {
logger.Fatalf("dump failed: %v", err)
}
if err := runCmd(ctx, logger, "user-program-import", map[string]string{
"USER_PROGRAM_UPDATE_DIR": updateDir,
"USER_PROGRAM_IMPORT_LOG_DIR": logDir,
"USER_PROGRAM_INFO_SCHEMA": schema,
}); err != nil {
logger.Fatalf("import failed: %v", err)
}
logger.Printf("sync completed (update_dir=%s, log_dir=%s, schema=%s)", updateDir, logDir, schema)
}
func runCmd(ctx context.Context, logger *log.Logger, command string, extraEnv map[string]string) error {
cmd := exec.CommandContext(ctx, command)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = os.Environ()
for k, v := range extraEnv {
if v == "" {
continue
}
cmd.Env = append(cmd.Env, k+"="+v)
}
logger.Printf("running %s", filepath.Base(command))
return cmd.Run()
}
func ensureDir(path string, logger *log.Logger) {
if err := os.MkdirAll(path, 0o755); err != nil {
logger.Fatalf("failed to create dir %s: %v", path, err)
}
}
func env(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}

View File

@@ -7,9 +7,9 @@ services:
db:
condition: service_healthy
ports:
- "${PORT:-8080}:8080"
- "${SERVICE_PORT:-8080}:8080"
environment:
- PORT=${PORT:-8080}
- SERVICE_PORT=${SERVICE_PORT:-8080}
- GEOIP_DB_PATH=${GEOIP_DB_PATH:-/app/initial_data/GeoLite2-City.mmdb}
- GEOIP_BACKEND=${GEOIP_BACKEND:-mmdb}
- GEOIP_LOADER_TIMEOUT=${GEOIP_LOADER_TIMEOUT:-30m}

2
go.mod
View File

@@ -3,6 +3,7 @@ module geoip-rest
go 1.25
require (
github.com/go-sql-driver/mysql v1.8.1
github.com/gofiber/fiber/v2 v2.52.8
github.com/jackc/pgx/v5 v5.7.6
github.com/oschwald/geoip2-golang v1.9.0
@@ -11,6 +12,7 @@ require (
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect

4
go.sum
View File

@@ -1,8 +1,12 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=

View File

@@ -12,9 +12,10 @@ import (
)
type Config struct {
CronExpr string
ScriptPath string
Logger *log.Logger
CronExpr string
Command string
Args []string
Logger *log.Logger
}
type Scheduler struct {
@@ -29,18 +30,14 @@ func Start(cfg Config) (*Scheduler, error) {
if cfg.CronExpr == "" {
return nil, errors.New("CronExpr is required")
}
if cfg.ScriptPath == "" {
return nil, errors.New("ScriptPath is required")
if cfg.Command == "" {
return nil, errors.New("Command is required")
}
if cfg.Logger == nil {
cfg.Logger = log.Default()
}
if _, err := os.Stat(cfg.ScriptPath); err != nil {
return nil, err
}
kst, err := time.LoadLocation("Asia/Seoul")
if err != nil {
kst = time.FixedZone("KST", 9*60*60)
@@ -54,12 +51,12 @@ func Start(cfg Config) (*Scheduler, error) {
c := cron.New(cron.WithLocation(kst), cron.WithParser(parser))
c.Schedule(spec, cron.FuncJob(func() {
runScript(cfg.Logger, cfg.ScriptPath)
runCommand(cfg.Logger, cfg.Command, cfg.Args...)
}))
c.Start()
cfg.Logger.Printf("scheduler started with cron=%s script=%s tz=%s", cfg.CronExpr, cfg.ScriptPath, kst)
cfg.Logger.Printf("scheduler started with cron=%s command=%s args=%v tz=%s", cfg.CronExpr, cfg.Command, cfg.Args, kst)
return &Scheduler{
cron: c,
@@ -75,11 +72,11 @@ func (s *Scheduler) Stop() context.Context {
return s.cron.Stop()
}
func runScript(logger *log.Logger, script string) {
func runCommand(logger *log.Logger, command string, args ...string) {
start := time.Now()
logger.Printf("scheduler: running %s", script)
logger.Printf("scheduler: running %s %v", command, args)
cmd := exec.Command("/bin/bash", script)
cmd := exec.Command(command, args...)
cmd.Env = os.Environ()
out, err := cmd.CombinedOutput()
duration := time.Since(start)
@@ -88,8 +85,8 @@ func runScript(logger *log.Logger, script string) {
logger.Printf("scheduler: output:\n%s", string(out))
}
if err != nil {
logger.Printf("scheduler: %s failed after %s: %v", script, duration, err)
logger.Printf("scheduler: %s failed after %s: %v", command, duration, err)
return
}
logger.Printf("scheduler: %s completed in %s", script, duration)
logger.Printf("scheduler: %s completed in %s", command, duration)
}

View File

@@ -1,67 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
LOG_DIR="${USER_PROGRAM_IMPORT_LOG_DIR:-/app/log}"
UPDATE_DIR="${USER_PROGRAM_UPDATE_DIR:-/app/update_data}"
SCHEMA="${USER_PROGRAM_INFO_SCHEMA:-public}"
CSV_DIR="${USER_PROGRAM_INFO_CSV_DIR:-/app/initial_data}"
MYSQL_HOST="${USER_PROGRAM_INFO_HOST:?USER_PROGRAM_INFO_HOST is required}"
MYSQL_PORT="${USER_PROGRAM_INFO_PORT:-3306}"
MYSQL_USER="${USER_PROGRAM_INFO_USERNAME:?USER_PROGRAM_INFO_USERNAME is required}"
MYSQL_PASS="${USER_PROGRAM_INFO_PASSWORD:?USER_PROGRAM_INFO_PASSWORD is required}"
MYSQL_DB="${USER_PROGRAM_INFO_DB:-user_program_info}"
MYSQL_TABLE="${USER_PROGRAM_INFO_TABLE:-user_program_info}"
mkdir -p "${LOG_DIR}" "${UPDATE_DIR}"
# Target date: yesterday in KST unless USER_PROGRAM_TARGET_DATE=YYYY-MM-DD is provided.
TARGET_DATE="${USER_PROGRAM_TARGET_DATE:-$(TZ=Asia/Seoul date -d 'yesterday' +%Y-%m-%d)}"
TARGET_DATE_COMPACT="${TARGET_DATE//-/}"
OUT_FILE="${UPDATE_DIR}/user_program_info_${TARGET_DATE_COMPACT}.csv"
TMP_FILE="${OUT_FILE}.tmp"
QUERY=$(cat <<SQL
SET time_zone = '+00:00';
SELECT
id,
product_name,
login_id,
user_employee_id,
login_version,
login_public_ip,
login_local_ip,
user_company,
user_department,
user_position,
user_login_time,
created_at,
user_family_flag
FROM ${MYSQL_TABLE}
WHERE DATE(CONVERT_TZ(created_at, '+00:00', '+09:00')) = '${TARGET_DATE}';
SQL
)
echo "[scheduler] dumping data for ${TARGET_DATE} to ${OUT_FILE}"
mysql --host="${MYSQL_HOST}" --port="${MYSQL_PORT}" --user="${MYSQL_USER}" --password="${MYSQL_PASS}" \
--database="${MYSQL_DB}" --batch --raw --silent --skip-column-names -e "${QUERY}" \
| python - <<'PY'
import csv, sys, os
out_path = os.environ["TMP_FILE"]
writer = csv.writer(open(out_path, "w", newline=""))
writer.writerow([
"id","product_name","login_id","user_employee_id","login_version",
"login_public_ip","login_local_ip","user_company","user_department",
"user_position","user_login_time","created_at","user_family_flag",
])
for line in sys.stdin:
row = line.rstrip("\n").split("\t")
writer.writerow(row)
PY
mv "${TMP_FILE}" "${OUT_FILE}"
echo "[scheduler] running import for ${OUT_FILE}"
DATABASE_URL="${DATABASE_URL:?DATABASE_URL is required}" USER_PROGRAM_UPDATE_DIR="${UPDATE_DIR}" USER_PROGRAM_IMPORT_LOG_DIR="${LOG_DIR}" \
user-program-import

View File

@@ -13,6 +13,10 @@
- [x] `user_program_info_replica` DDL/CSV 임포터 추가 (`id bigint`, 텍스트 컬럼, timestamp KST 파싱, bool 플래그) 완료: 2025-12-09 18:32 KST
- [x] 초기/일간 CSV 디렉토리 기반 임포트 + 로그 파일 기록(`log/`), upsert 로직 업데이트 완료: 2025-12-09 19:06 KST
- [x] Fiber 프로세스 내 cron 스케줄러 추가(전일 덤프 스크립트 실행 + update_data 적용, KST cron 지원) 완료: 2025-12-09 19:28 KST
- [x] MySQL CLI 의존성 제거, Go 기반 덤퍼(`cmd/user_program_dump`) 추가 및 `scripts/dump_and_import.sh`에서 사용하도록 변경 완료: 2025-12-10 09:34 KST
- [x] 스케줄러 토글 env(`USER_PROGRAM_CRON_ENABLE`) 추가, true일 때만 크론 구동하도록 변경 완료: 2025-12-10 09:45 KST
- [x] 크론 표현식 env(`USER_PROGRAM_CRON`) 제거, 코드에 KST 00:05 고정 스케줄 적용 완료: 2025-12-10 09:56 KST
- [x] bash 스크립트 의존 없이 Go CLI(`user-program-sync`)로 덤프+임포트 수행, 스케줄러가 해당 CLI를 직접 호출하도록 변경 완료: 2025-12-10 09:50 KST
## 진행 예정
- [x] PostgreSQL 전용 Docker 이미지(또는 build 단계)에서 `maxminddb_fdw` 설치 후 `GeoLite2-City.mmdb` 볼륨을 `/data`로 마운트하는 `postgres` 서비스 추가 및 5432 외부 노출